casacore
Loading...
Searching...
No Matches

More...

#include <StArrayFile.h>

Public Member Functions

 StManArrayFile (const String &name, ByteIO::OpenOption, uInt version=0, Bool bigEndian=True, uInt bufferSize=0, const std::shared_ptr< MultiFileBase > &=std::shared_ptr< MultiFileBase >())
 Construct the object and attach it to the give file.
 
 ~StManArrayFile ()
 Close the possibly opened file.
 
Bool flush (Bool fsync)
 Flush and optionally fsync the data.
 
void reopenRW ()
 Reopen the file for read/write access.
 
void resync ()
 Resync the file (i.e.
 
Int64 length ()
 Return the current file length (merely a debug tool).
 
uInt putShape (const IPosition &shape, Int64 &fileOffset, const Bool *dummy)
 Put the array shape and store its file offset into the offset argument.
 
uInt putShape (const IPosition &shape, Int64 &fileOffset, const Char *dummy)
 
uInt putShape (const IPosition &shape, Int64 &fileOffset, const uChar *dummy)
 
uInt putShape (const IPosition &shape, Int64 &fileOffset, const Short *dummy)
 
uInt putShape (const IPosition &shape, Int64 &fileOffset, const uShort *dummy)
 
uInt putShape (const IPosition &shape, Int64 &fileOffset, const Int *dummy)
 
uInt putShape (const IPosition &shape, Int64 &fileOffset, const uInt *dummy)
 
uInt putShape (const IPosition &shape, Int64 &fileOffset, const Int64 *dummy)
 
uInt putShape (const IPosition &shape, Int64 &fileOffset, const uInt64 *dummy)
 
uInt putShape (const IPosition &shape, Int64 &fileOffset, const Float *dummy)
 
uInt putShape (const IPosition &shape, Int64 &fileOffset, const Double *dummy)
 
uInt putShape (const IPosition &shape, Int64 &fileOffset, const Complex *dummy)
 
uInt putShape (const IPosition &shape, Int64 &fileOffset, const DComplex *dummy)
 
uInt putShape (const IPosition &shape, Int64 &fileOffset, const String *dummy)
 
uInt getRefCount (Int64 offset)
 Get the reference count.
 
void putRefCount (uInt refCount, Int64 offset)
 Put the reference count.
 
void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Bool *)
 Put nr elements at the given file offset and array offset.
 
void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Char *)
 
void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uChar *)
 
void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Short *)
 
void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uShort *)
 
void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Int *)
 
void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uInt *)
 
void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Int64 *)
 
void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uInt64 *)
 
void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Float *)
 
void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Double *)
 
void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Complex *)
 
void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const DComplex *)
 
void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const String *)
 
uInt getShape (Int64 fileOffset, IPosition &shape)
 Get the shape at the given file offset.
 
void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Bool *)
 Get nr elements at the given file offset and array offset.
 
void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Char *)
 
void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uChar *)
 
void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Short *)
 
void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uShort *)
 
void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Int *)
 
void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uInt *)
 
void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Int64 *)
 
void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uInt64 *)
 
void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Float *)
 
void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Double *)
 
void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Complex *)
 
void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, DComplex *)
 
void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, String *)
 
void copyArrayBool (Int64 to, Int64 from, uInt64 nr)
 Copy the array with nr elements from one file offset to another.
 
void copyArrayChar (Int64 to, Int64 from, uInt64 nr)
 
void copyArrayuChar (Int64 to, Int64 from, uInt64 nr)
 
void copyArrayShort (Int64 to, Int64 from, uInt64 nr)
 
void copyArrayuShort (Int64 to, Int64 from, uInt64 nr)
 
void copyArrayInt (Int64 to, Int64 from, uInt64 nr)
 
void copyArrayuInt (Int64 to, Int64 from, uInt64 nr)
 
void copyArrayInt64 (Int64 to, Int64 from, uInt64 nr)
 
void copyArrayuInt64 (Int64 to, Int64 from, uInt64 nr)
 
void copyArrayFloat (Int64 to, Int64 from, uInt64 nr)
 
void copyArrayDouble (Int64 to, Int64 from, uInt64 nr)
 
void copyArrayComplex (Int64 to, Int64 from, uInt64 nr)
 
void copyArrayDComplex (Int64 to, Int64 from, uInt64 nr)
 
void copyArrayString (Int64 to, Int64 from, uInt64 nr)
 

Private Member Functions

uInt put (const Int &)
 Put a single value at the current file offset.
 
uInt put (const uInt &)
 
uInt putRes (const IPosition &shape, Int64 &fileOffset, float lenElem)
 Put the array shape at the end of the file and reserve space for nr elements (each lenElem bytes long).
 
uInt get (Int &)
 Get a single value at the current file offset.
 
uInt get (uInt &)
 
void copyData (Int64 to, Int64 from, uInt64 length)
 Copy data with the given length from one file offset to another.
 
void setpos (Int64 offset)
 Position the file on the given offset.
 

Private Attributes

std::shared_ptr< ByteIOfile_p
 
std::shared_ptr< TypeIOiofil_p
 
Int64 leng_p
 
uInt version_p
 
Bool swput_p
 
Bool hasPut_p
 
uInt sizeChar_p
 
uInt sizeuChar_p
 
uInt sizeShort_p
 
uInt sizeuShort_p
 
uInt sizeInt_p
 
uInt sizeuInt_p
 
uInt sizeInt64_p
 
uInt sizeuInt64_p
 
uInt sizeFloat_p
 
uInt sizeDouble_p
 

Detailed Description

Read/write array in external format for a storage manager

Intended use:

Internal

Review Status

Reviewed By:
UNKNOWN
Date Reviewed:
before2004/08/25

Prerequisite

  • ToLocal
  • FromLocal

Etymology

StManArrayFile is a class used by table storage managers to store indirect arrays in a file.

Synopsis

StManArrayFile is for use by the table storage manager, in particular to read/write indirectly stored arrays. Instead of holding the data in memory, they are written directly into a file. It also allows to access a part of an array, which is needed for the table system to access an array section. It does not use a cache of its own, but it is relying on the underlying system routines to cache and buffer adequately.

This class could in principle also be used for other array purposes, for example, to implement a paged array class for really huge arrays.

An StManArrayFile object is connected to one file. It is possible to hold multiple arrays in the file, each with its own shape. An array is stored as its shape followed by the actual data (all in little or big endian format). An array of strings is written as an array of offsets pointing to the actual strings. When a string gets a new value, the new value is written at the end of the file and the file space with the old value is lost.

Currently only the basic types are supported, but arbitrary types could also be supported by writing/reading an element in the normal way into the AipsIO buffer. It would only require that AipsIO would contain a function to get its buffers and to restart them.

Example

void writeArray (const Array<Bool>& array)
{
// Construct object and update file StArray.dat.
StManArrayFile arrayFile("StArray.dat, ByteIO::New);
// Reserve space for an array with the given shape and data type.
// This writes the shape at the end of the file and reserves
// space the hold the entire Bool array.
// It fills in the file offset where the shape is stored
// and returns the length of the shape in the file.
Int64 offset;
uInt shapeLength = arrayFile.putShape (array.shape(), offset, static_cast<Bool*>(0));
// Now put the actual array.
// This has to be put at the returned file offset plus the length
// of the shape in the file.
Bool deleteIt;
const Bool* dataPtr = array.getStorage (deleteIt);
arrayFile.put (offset+shapeLength, 0, array.nelements(), dataPtr);
array.freeStorage (dataPtr, deleteIt);
}
TableExprNode array(const TableExprNode &values, const TableExprNodeSet &shape)
Create an array of the given shape and fill it with the values.
Definition ExprNode.h:1933

Motivation

The AipsIO class was not suitable for indirect table arrays, because it uses memory to hold the data. Furthermore it is not possible to access part of the data in AipsIO.

To Do

  • implement long double
  • support arbitrary types
  • when rewriting a string value, use the current file space if it fits

Definition at line 127 of file StArrayFile.h.

Constructor & Destructor Documentation

◆ StManArrayFile()

casacore::StManArrayFile::StManArrayFile ( const String & name,
ByteIO::OpenOption ,
uInt version = 0,
Bool bigEndian = True,
uInt bufferSize = 0,
const std::shared_ptr< MultiFileBase > & = std::shared_ptr< MultiFileBase >() )

Construct the object and attach it to the give file.

The OpenOption determines how the file is opened (e.g. ByteIO::New for a new file). The buffersize is used to allocate a buffer of a proper size for the underlying filebuf object (see iostream package). A bufferSize 0 means using the default size (currently 65536).

◆ ~StManArrayFile()

casacore::StManArrayFile::~StManArrayFile ( )

Close the possibly opened file.

Member Function Documentation

◆ copyArrayBool()

void casacore::StManArrayFile::copyArrayBool ( Int64 to,
Int64 from,
uInt64 nr )

Copy the array with nr elements from one file offset to another.

◆ copyArrayChar()

void casacore::StManArrayFile::copyArrayChar ( Int64 to,
Int64 from,
uInt64 nr )

◆ copyArrayComplex()

void casacore::StManArrayFile::copyArrayComplex ( Int64 to,
Int64 from,
uInt64 nr )

◆ copyArrayDComplex()

void casacore::StManArrayFile::copyArrayDComplex ( Int64 to,
Int64 from,
uInt64 nr )

◆ copyArrayDouble()

void casacore::StManArrayFile::copyArrayDouble ( Int64 to,
Int64 from,
uInt64 nr )

◆ copyArrayFloat()

void casacore::StManArrayFile::copyArrayFloat ( Int64 to,
Int64 from,
uInt64 nr )

◆ copyArrayInt()

void casacore::StManArrayFile::copyArrayInt ( Int64 to,
Int64 from,
uInt64 nr )

◆ copyArrayInt64()

void casacore::StManArrayFile::copyArrayInt64 ( Int64 to,
Int64 from,
uInt64 nr )

◆ copyArrayShort()

void casacore::StManArrayFile::copyArrayShort ( Int64 to,
Int64 from,
uInt64 nr )

◆ copyArrayString()

void casacore::StManArrayFile::copyArrayString ( Int64 to,
Int64 from,
uInt64 nr )

◆ copyArrayuChar()

void casacore::StManArrayFile::copyArrayuChar ( Int64 to,
Int64 from,
uInt64 nr )

◆ copyArrayuInt()

void casacore::StManArrayFile::copyArrayuInt ( Int64 to,
Int64 from,
uInt64 nr )

◆ copyArrayuInt64()

void casacore::StManArrayFile::copyArrayuInt64 ( Int64 to,
Int64 from,
uInt64 nr )

◆ copyArrayuShort()

void casacore::StManArrayFile::copyArrayuShort ( Int64 to,
Int64 from,
uInt64 nr )

◆ copyData()

void casacore::StManArrayFile::copyData ( Int64 to,
Int64 from,
uInt64 length )
private

Copy data with the given length from one file offset to another.

◆ flush()

Bool casacore::StManArrayFile::flush ( Bool fsync)

Flush and optionally fsync the data.

It returns True when any data was written since the last flush.

◆ get() [1/16]

uInt casacore::StManArrayFile::get ( Int & value)
inlineprivate

Get a single value at the current file offset.

It returns the length of the value in the file.

Definition at line 334 of file StArrayFile.h.

◆ get() [2/16]

void casacore::StManArrayFile::get ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
Bool *  )

Get nr elements at the given file offset and array offset.

The file offset of the first array element is the file offset of the shape plus the length of the shape in the file. The array offset is counted in number of elements. It can be used to get only a (contiguous) section of the array.

◆ get() [3/16]

void casacore::StManArrayFile::get ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
Char *  )

◆ get() [4/16]

void casacore::StManArrayFile::get ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
Complex *  )

◆ get() [5/16]

void casacore::StManArrayFile::get ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
DComplex *  )

◆ get() [6/16]

void casacore::StManArrayFile::get ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
Double *  )

◆ get() [7/16]

void casacore::StManArrayFile::get ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
Float *  )

◆ get() [8/16]

void casacore::StManArrayFile::get ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
Int *  )

◆ get() [9/16]

void casacore::StManArrayFile::get ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
Int64 *  )

◆ get() [10/16]

void casacore::StManArrayFile::get ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
Short *  )

◆ get() [11/16]

void casacore::StManArrayFile::get ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
String *  )

◆ get() [12/16]

void casacore::StManArrayFile::get ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
uChar *  )

◆ get() [13/16]

void casacore::StManArrayFile::get ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
uInt *  )

◆ get() [14/16]

void casacore::StManArrayFile::get ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
uInt64 *  )

◆ get() [15/16]

void casacore::StManArrayFile::get ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
uShort *  )

◆ get() [16/16]

uInt casacore::StManArrayFile::get ( uInt & value)
inlineprivate

Definition at line 338 of file StArrayFile.h.

◆ getRefCount()

uInt casacore::StManArrayFile::getRefCount ( Int64 offset)

Get the reference count.

◆ getShape()

uInt casacore::StManArrayFile::getShape ( Int64 fileOffset,
IPosition & shape )

Get the shape at the given file offset.

It will reshape the IPosition vector when needed. It returns the length of the shape in the file.

◆ length()

Int64 casacore::StManArrayFile::length ( )
inline

Return the current file length (merely a debug tool).

Definition at line 156 of file StArrayFile.h.

◆ put() [1/16]

uInt casacore::StManArrayFile::put ( const Int & value)
inlineprivate

Put a single value at the current file offset.

It returns the length of the value in the file.

Definition at line 324 of file StArrayFile.h.

◆ put() [2/16]

uInt casacore::StManArrayFile::put ( const uInt & value)
inlineprivate

Definition at line 329 of file StArrayFile.h.

◆ put() [3/16]

void casacore::StManArrayFile::put ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
const Bool *  )

Put nr elements at the given file offset and array offset.

The file offset of the first array element is the file offset of the shape plus the length of the shape in the file. The array offset is counted in number of elements. It can be used to put only a (contiguous) section of the array.

◆ put() [4/16]

void casacore::StManArrayFile::put ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
const Char *  )

◆ put() [5/16]

void casacore::StManArrayFile::put ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
const Complex *  )

◆ put() [6/16]

void casacore::StManArrayFile::put ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
const DComplex *  )

◆ put() [7/16]

void casacore::StManArrayFile::put ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
const Double *  )

◆ put() [8/16]

void casacore::StManArrayFile::put ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
const Float *  )

◆ put() [9/16]

void casacore::StManArrayFile::put ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
const Int *  )

◆ put() [10/16]

void casacore::StManArrayFile::put ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
const Int64 *  )

◆ put() [11/16]

void casacore::StManArrayFile::put ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
const Short *  )

◆ put() [12/16]

void casacore::StManArrayFile::put ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
const String *  )

◆ put() [13/16]

void casacore::StManArrayFile::put ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
const uChar *  )

◆ put() [14/16]

void casacore::StManArrayFile::put ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
const uInt *  )

◆ put() [15/16]

void casacore::StManArrayFile::put ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
const uInt64 *  )

◆ put() [16/16]

void casacore::StManArrayFile::put ( Int64 fileOffset,
Int64 arrayOffset,
uInt64 nr,
const uShort *  )

◆ putRefCount()

void casacore::StManArrayFile::putRefCount ( uInt refCount,
Int64 offset )

Put the reference count.

An exception is thrown if a value other than 1 is put for version 0.

◆ putRes()

uInt casacore::StManArrayFile::putRes ( const IPosition & shape,
Int64 & fileOffset,
float lenElem )
private

Put the array shape at the end of the file and reserve space for nr elements (each lenElem bytes long).

It fills the file offset of the shape. It returns the length of the shape in the file.

◆ putShape() [1/14]

uInt casacore::StManArrayFile::putShape ( const IPosition & shape,
Int64 & fileOffset,
const Bool * dummy )

Put the array shape and store its file offset into the offset argument.

Reserve file space for the associated array. The length of the shape part in the file is returned. The file offset plus the shape length is the starting offset of the actual array data (which can be used by get and put). Space is reserved to store the reference count.

◆ putShape() [2/14]

uInt casacore::StManArrayFile::putShape ( const IPosition & shape,
Int64 & fileOffset,
const Char * dummy )

◆ putShape() [3/14]

uInt casacore::StManArrayFile::putShape ( const IPosition & shape,
Int64 & fileOffset,
const Complex * dummy )

◆ putShape() [4/14]

uInt casacore::StManArrayFile::putShape ( const IPosition & shape,
Int64 & fileOffset,
const DComplex * dummy )

◆ putShape() [5/14]

uInt casacore::StManArrayFile::putShape ( const IPosition & shape,
Int64 & fileOffset,
const Double * dummy )

◆ putShape() [6/14]

uInt casacore::StManArrayFile::putShape ( const IPosition & shape,
Int64 & fileOffset,
const Float * dummy )

◆ putShape() [7/14]

uInt casacore::StManArrayFile::putShape ( const IPosition & shape,
Int64 & fileOffset,
const Int * dummy )

◆ putShape() [8/14]

uInt casacore::StManArrayFile::putShape ( const IPosition & shape,
Int64 & fileOffset,
const Int64 * dummy )

◆ putShape() [9/14]

uInt casacore::StManArrayFile::putShape ( const IPosition & shape,
Int64 & fileOffset,
const Short * dummy )

◆ putShape() [10/14]

uInt casacore::StManArrayFile::putShape ( const IPosition & shape,
Int64 & fileOffset,
const String * dummy )

◆ putShape() [11/14]

uInt casacore::StManArrayFile::putShape ( const IPosition & shape,
Int64 & fileOffset,
const uChar * dummy )

◆ putShape() [12/14]

uInt casacore::StManArrayFile::putShape ( const IPosition & shape,
Int64 & fileOffset,
const uInt * dummy )

◆ putShape() [13/14]

uInt casacore::StManArrayFile::putShape ( const IPosition & shape,
Int64 & fileOffset,
const uInt64 * dummy )

◆ putShape() [14/14]

uInt casacore::StManArrayFile::putShape ( const IPosition & shape,
Int64 & fileOffset,
const uShort * dummy )

◆ reopenRW()

void casacore::StManArrayFile::reopenRW ( )
inline

Reopen the file for read/write access.

Definition at line 320 of file StArrayFile.h.

References file_p.

◆ resync()

void casacore::StManArrayFile::resync ( )

Resync the file (i.e.

clear possible cache information).

◆ setpos()

void casacore::StManArrayFile::setpos ( Int64 offset)
private

Position the file on the given offset.

Member Data Documentation

◆ file_p

std::shared_ptr<ByteIO> casacore::StManArrayFile::file_p
private

Definition at line 275 of file StArrayFile.h.

Referenced by reopenRW().

◆ hasPut_p

Bool casacore::StManArrayFile::hasPut_p
private

Definition at line 280 of file StArrayFile.h.

◆ iofil_p

std::shared_ptr<TypeIO> casacore::StManArrayFile::iofil_p
private

Definition at line 276 of file StArrayFile.h.

◆ leng_p

Int64 casacore::StManArrayFile::leng_p
private

Definition at line 277 of file StArrayFile.h.

◆ sizeChar_p

uInt casacore::StManArrayFile::sizeChar_p
private

Definition at line 281 of file StArrayFile.h.

◆ sizeDouble_p

uInt casacore::StManArrayFile::sizeDouble_p
private

Definition at line 290 of file StArrayFile.h.

◆ sizeFloat_p

uInt casacore::StManArrayFile::sizeFloat_p
private

Definition at line 289 of file StArrayFile.h.

◆ sizeInt64_p

uInt casacore::StManArrayFile::sizeInt64_p
private

Definition at line 287 of file StArrayFile.h.

◆ sizeInt_p

uInt casacore::StManArrayFile::sizeInt_p
private

Definition at line 285 of file StArrayFile.h.

◆ sizeShort_p

uInt casacore::StManArrayFile::sizeShort_p
private

Definition at line 283 of file StArrayFile.h.

◆ sizeuChar_p

uInt casacore::StManArrayFile::sizeuChar_p
private

Definition at line 282 of file StArrayFile.h.

◆ sizeuInt64_p

uInt casacore::StManArrayFile::sizeuInt64_p
private

Definition at line 288 of file StArrayFile.h.

◆ sizeuInt_p

uInt casacore::StManArrayFile::sizeuInt_p
private

Definition at line 286 of file StArrayFile.h.

◆ sizeuShort_p

uInt casacore::StManArrayFile::sizeuShort_p
private

Definition at line 284 of file StArrayFile.h.

◆ swput_p

Bool casacore::StManArrayFile::swput_p
private

Definition at line 279 of file StArrayFile.h.

◆ version_p

uInt casacore::StManArrayFile::version_p
private

Definition at line 278 of file StArrayFile.h.


The documentation for this class was generated from the following file: