From 0b950dd18e4a2ac7d051210a0fb40114a35792b2 Mon Sep 17 00:00:00 2001 From: Clifford Yapp <238416+starseeker@users.noreply.github.com> Date: Sat, 1 Aug 2026 22:22:05 -0400 Subject: [PATCH 01/11] cmake: declare target link interfaces explicitly Use modern target_link_libraries scopes so libraries publish their transitive requirements and executables keep theirs private. This preserves schema test linkage with current CMake versions. --- cmake/SC_Targets.cmake | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmake/SC_Targets.cmake b/cmake/SC_Targets.cmake index 8218fad6e..4daa44179 100644 --- a/cmake/SC_Targets.cmake +++ b/cmake/SC_Targets.cmake @@ -22,6 +22,9 @@ macro(SC_ADDEXEC execname) message(SEND_ERROR "SC_ADDEXEC usage error - expected STATIC LINK_LIBRARIES targets (${_lib})") endif() endif() + # Executables consume their dependencies but do not publish an + # interface to downstream targets. + target_link_libraries(${execname} PRIVATE ${_lib}) endforeach() target_link_libraries(${execname} PRIVATE ${${_arg_prefix}_LINK_LIBRARIES}) endif() @@ -74,6 +77,11 @@ macro(SC_ADDLIB _addlib_target) message(SEND_ERROR "SC_ADDLIB usage error - expected (static) LINK_LIBRARIES targets (${_lib})") endif() endif() + # Schema libraries are consumed directly by generated test programs. + # Publish their link requirements explicitly; the legacy unscoped + # signature does not reliably provide a transitive interface with + # current CMake versions. + target_link_libraries(${_addlib_target} PUBLIC ${_lib}) endforeach() target_link_libraries(${_addlib_target} ${_lib}) endif() @@ -93,4 +101,3 @@ endmacro() # indent-tabs-mode: t # End: # ex: shiftwidth=2 tabstop=8 - From 9b233ac6ca52ad192e1c9927c349cb400bcb0a4a Mon Sep 17 00:00:00 2001 From: Clifford Yapp <238416+starseeker@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:55:30 -0400 Subject: [PATCH 02/11] lazy: add indexed Part 21 instance materialization Add a lazy Part 21 reader and instance manager that index records and resolve SDAI instances on demand. Include diagnostics, cache management, reference indexes, and baseline coverage for lazy reads. --- include/cllazyfile/CMakeLists.txt | 2 +- include/cllazyfile/headerSectionReader.h | 5 +- include/cllazyfile/instMgrHelper.h | 1 - include/cllazyfile/judyLArray.h | 4 +- include/cllazyfile/lazyFileReader.h | 9 +- include/cllazyfile/lazyInstMgr.h | 77 ++++- include/cllazyfile/lazyP21DataSectionReader.h | 5 +- include/cllazyfile/lazySupport.h | 128 ++++++++ include/cllazyfile/lazyTypes.h | 13 +- include/cllazyfile/sectionReader.h | 9 +- src/cllazyfile/CMakeLists.txt | 13 + src/cllazyfile/lazyDataSectionReader.cc | 4 +- src/cllazyfile/lazyFileReader.cc | 54 +++- src/cllazyfile/lazyInstMgr.cc | 297 ++++++++++++++++-- src/cllazyfile/lazyP21DataSectionReader.cc | 62 +++- src/cllazyfile/lazySupport.cc | 106 +++++++ src/cllazyfile/lazy_index_test.cc | 60 ++++ src/cllazyfile/lazy_test.cc | 29 +- src/cllazyfile/p21HeaderSectionReader.cc | 8 +- src/cllazyfile/sectionReader.cc | 40 ++- src/cllazyfile/test/lazy_index.exp | 25 ++ src/cllazyfile/test/lazy_index.stp | 17 + 22 files changed, 877 insertions(+), 91 deletions(-) create mode 100644 include/cllazyfile/lazySupport.h create mode 100644 src/cllazyfile/lazySupport.cc create mode 100644 src/cllazyfile/lazy_index_test.cc create mode 100644 src/cllazyfile/test/lazy_index.exp create mode 100644 src/cllazyfile/test/lazy_index.stp diff --git a/include/cllazyfile/CMakeLists.txt b/include/cllazyfile/CMakeLists.txt index 853aa98b9..488e6135d 100644 --- a/include/cllazyfile/CMakeLists.txt +++ b/include/cllazyfile/CMakeLists.txt @@ -5,6 +5,7 @@ set(LAZY_HDRS p21HeaderSectionReader.h lazyDataSectionReader.h lazyInstMgr.h + lazySupport.h lazyTypes.h sectionReader.h instMgrHelper.h @@ -24,4 +25,3 @@ install(FILES ${LAZY_HDRS} # indent-tabs-mode: t # End: # ex: shiftwidth=2 tabstop=8 - diff --git a/include/cllazyfile/headerSectionReader.h b/include/cllazyfile/headerSectionReader.h index e3a9f5725..2453503b9 100644 --- a/include/cllazyfile/headerSectionReader.h +++ b/include/cllazyfile/headerSectionReader.h @@ -27,12 +27,9 @@ class SC_LAZYFILE_EXPORT headerSectionReader: public sectionReader { } virtual ~headerSectionReader() { - //FIXME delete each instance?! maybe add to clear, since it iterates over everything already - //enum clearHow { rawData, deletePointers } - _headerInstances->clear(); + _headerInstances->clear( true ); delete _headerInstances; } }; #endif //HEADERSECTIONREADER_H - diff --git a/include/cllazyfile/instMgrHelper.h b/include/cllazyfile/instMgrHelper.h index 1af698160..daa046eb3 100644 --- a/include/cllazyfile/instMgrHelper.h +++ b/include/cllazyfile/instMgrHelper.h @@ -58,4 +58,3 @@ class SC_LAZYFILE_EXPORT instMgrAdapter: public InstMgrBase { #endif //INSTMGRHELPER_H - diff --git a/include/cllazyfile/judyLArray.h b/include/cllazyfile/judyLArray.h index b78dc0bb9..877420549 100644 --- a/include/cllazyfile/judyLArray.h +++ b/include/cllazyfile/judyLArray.h @@ -159,8 +159,8 @@ class judyLArray { * getLastValue() will return the entry before the one that was deleted * \sa isEmpty() */ - bool removeEntry( JudyKey * key ) { - if( judy_slot( _judyarray, key, _depth * JUDY_key_size ) ) { + bool removeEntry( JudyKey key ) { + if( judy_slot( _judyarray, reinterpret_cast( &key ), _depth * JUDY_key_size ) ) { _lastSlot = ( JudyValue * ) judy_del( _judyarray ); return true; } else { diff --git a/include/cllazyfile/lazyFileReader.h b/include/cllazyfile/lazyFileReader.h index 9a5fccb5e..97b53f1ea 100644 --- a/include/cllazyfile/lazyFileReader.h +++ b/include/cllazyfile/lazyFileReader.h @@ -37,6 +37,8 @@ class SC_LAZYFILE_EXPORT lazyFileReader { #endif fileTypeEnum _fileType; fileID _fileID; + lazyFileOffset _fileSize; + bool _valid; void initP21(); @@ -48,6 +50,12 @@ class SC_LAZYFILE_EXPORT lazyFileReader { fileID ID() const { return _fileID; } + lazyFileOffset fileSize() const { + return _fileSize; + } + bool valid() const { + return _valid; + } instancesLoaded_t * getHeaderInstances(); lazyFileReader( std::string fname, lazyInstMgr * i, fileID fid ); @@ -64,4 +72,3 @@ class SC_LAZYFILE_EXPORT lazyFileReader { }; #endif //LAZYFILEREADER_H - diff --git a/include/cllazyfile/lazyInstMgr.h b/include/cllazyfile/lazyInstMgr.h index e3445660a..63443a474 100644 --- a/include/cllazyfile/lazyInstMgr.h +++ b/include/cllazyfile/lazyInstMgr.h @@ -2,12 +2,15 @@ #define LAZYINSTMGR_H #include +#include #include +#include #include #include "cllazyfile/lazyDataSectionReader.h" #include "cllazyfile/lazyFileReader.h" #include "cllazyfile/lazyTypes.h" +#include "cllazyfile/lazySupport.h" #include "clstepcore/Registry.h" #include "sc_export.h" @@ -61,15 +64,37 @@ class SC_LAZYFILE_EXPORT lazyInstMgr { lazyFileReaderVec_t _files; + /** All indexed DATA instance IDs in deterministic file order. */ + instanceRefs _allInstances; + Registry * _headerRegistry, * _mainRegistry; + bool _ownsMainRegistry; ErrorDescriptor * _errors; unsigned long _lazyInstanceCount, _loadedInstanceCount; + uint64_t _cacheHighWater, _cacheHits, _cacheMisses, _materializations, _evictions; + uint64_t _activeBatches; int _longestTypeNameLen; std::string _longestTypeName; + std::map _pinCounts; + std::set _batchOwnedInstances; + std::set _permanentlyLoadedInstances; + size_t _batchLoadDepth; + + LazyProgressCallback _progressCallback; + LazyCancellationCallback _cancellationCallback; + LazyDiagnosticCallback _diagnosticCallback; + std::map _diagnosticCounts; + bool _cancelled; + instMgrAdapter * _ima; + friend class LazyInstanceBatch; + void releaseBatch( const std::vector & instances ); + SDAI_Application_instance * cachedInstance( instanceID id ); + std::vector dependencyClosure( const std::vector & roots ); + #ifdef _MSC_VER #pragma warning( pop ) #endif @@ -77,7 +102,7 @@ class SC_LAZYFILE_EXPORT lazyInstMgr { public: lazyInstMgr(); ~lazyInstMgr(); - void openFile( std::string fname ); + bool openFile( std::string fname ); void addLazyInstance( namedLazyInstance inst ); InstMgrBase * getAdapter() { @@ -91,19 +116,25 @@ class SC_LAZYFILE_EXPORT lazyInstMgr { instanceRefs_t * getRevRefs() { return & _revInstanceRefs; } + LazyInstanceIdView instancesByType( std::string type, bool caseSensitive = false ); + LazyInstanceIdView allInstances() const { + return LazyInstanceIdView( &_allInstances ); + } + LazyInstanceIdView forwardReferences( instanceID id ); + LazyInstanceIdView reverseReferences( instanceID id ); /// returns a vector containing the instances that match `type` instanceTypes_t::cvector * getInstances( std::string type, bool caseSensitive = false ) { /*const*/ if( !caseSensitive ) { std::string::iterator it = type.begin(); for( ; it != type.end(); ++it ) { - *it = toupper( *it ); + *it = static_cast( toupper( static_cast( *it ) ) ); } } return _instanceTypes->find( type.c_str() ); } /// get the number of instances of a certain type - unsigned int countInstances( std::string type ) { - instanceTypes_t::cvector * v = _instanceTypes->find( type.c_str() ); + unsigned int countInstances( std::string type, bool caseSensitive = false ) { + instanceTypes_t::cvector * v = getInstances( type, caseSensitive ); if( !v ) { return 0; } @@ -123,6 +154,28 @@ class SC_LAZYFILE_EXPORT lazyInstMgr { return _loadedInstanceCount; } + LazyCacheStatistics cacheStatistics() const; + + void setProgressCallback( const LazyProgressCallback & callback ) { + _progressCallback = callback; + } + void setCancellationCallback( const LazyCancellationCallback & callback ) { + _cancellationCallback = callback; + } + void setDiagnosticCallback( const LazyDiagnosticCallback & callback ) { + _diagnosticCallback = callback; + } + bool cancelled() const { + return _cancelled; + } + void observeScan( fileID file, lazyFileOffset offset, lazyFileOffset fileSize ); + void emitDiagnostic( LazyDiagnostic diagnostic ); + uint64_t diagnosticCount( const std::string & key ) const; + const std::map & diagnosticCounts() const { + return _diagnosticCounts; + } + void validateReferences(); + /// get the number of data sections that have been identified unsigned int countDataSections() { return _dataSections.size(); @@ -130,7 +183,9 @@ class SC_LAZYFILE_EXPORT lazyInstMgr { ///builds the registry using the given initFunct const Registry * initRegistry( CF_init initFunct ) { - setRegistry( new Registry( initFunct ) ); + assert( _mainRegistry == 0 ); + _mainRegistry = new Registry( initFunct ); + _ownsMainRegistry = true; return _mainRegistry; } @@ -138,6 +193,7 @@ class SC_LAZYFILE_EXPORT lazyInstMgr { void setRegistry( Registry * reg ) { assert( _mainRegistry == 0 ); _mainRegistry = reg; + _ownsMainRegistry = false; } const Registry * getHeaderRegistry() const { @@ -168,6 +224,9 @@ class SC_LAZYFILE_EXPORT lazyInstMgr { */ SDAI_Application_instance * loadInstance( instanceID id, bool reSeek = false ); + LazyInstanceBatch loadBatch( instanceID root ); + LazyInstanceBatch loadBatch( const std::vector & roots ); + //list all instances that one instance depends on (recursive) instanceSet * instanceDependencies( instanceID id ); bool isLoaded( instanceID id ) { @@ -183,11 +242,8 @@ class SC_LAZYFILE_EXPORT lazyInstMgr { std::cerr << "Error at " << __FILE__ << ":" << __LINE__ << " - multiple instances (" << cv->size() << ") with one instanceID (" << id << ") not supported yet." << std::endl; return 0; } - positionAndSection ps = cv->at( 0 ); - //extract p, s, call - long int off = ps & 0xFFFFFFFFFFFFULL; - sectionID sid = ps >> 48; - return _dataSections[sid]->getType( off ); + instancePosition pos = cv->at( 0 ); + return _dataSections[pos.section]->getType( pos.begin ); } std::cerr << "Error at " << __FILE__ << ":" << __LINE__ << " - instanceID " << id << " not found." << std::endl; return 0; @@ -218,4 +274,3 @@ class SC_LAZYFILE_EXPORT lazyInstMgr { }; #endif //LAZYINSTMGR_H - diff --git a/include/cllazyfile/lazyP21DataSectionReader.h b/include/cllazyfile/lazyP21DataSectionReader.h index c14376fc0..9086630ec 100644 --- a/include/cllazyfile/lazyP21DataSectionReader.h +++ b/include/cllazyfile/lazyP21DataSectionReader.h @@ -10,9 +10,7 @@ class SC_LAZYFILE_EXPORT lazyP21DataSectionReader: public lazyDataSectionReader public: lazyP21DataSectionReader( lazyFileReader * parent, std::ifstream & file, std::streampos start, sectionID sid ); - void findSectionStart() { - _sectionStart = findNormalString( "DATA", true ); - } + void findSectionStart(); /** gets information (start, end, name, etc) about the next * instance in the file and returns it in a namedLazyInstance * \sa p21HeaderSectionReader::nextInstance() @@ -22,4 +20,3 @@ class SC_LAZYFILE_EXPORT lazyP21DataSectionReader: public lazyDataSectionReader }; #endif //LAZYP21DATASECTIONREADER_H - diff --git a/include/cllazyfile/lazySupport.h b/include/cllazyfile/lazySupport.h new file mode 100644 index 000000000..ce398fea7 --- /dev/null +++ b/include/cllazyfile/lazySupport.h @@ -0,0 +1,128 @@ +#ifndef LAZYSUPPORT_H +#define LAZYSUPPORT_H + +#include +#include +#include +#include +#include + +#include "cllazyfile/lazyTypes.h" +#include "sc_export.h" + +class lazyInstMgr; +class SDAI_Application_instance; + +enum LazyDiagnosticSeverity { + LAZY_DIAGNOSTIC_INFO, + LAZY_DIAGNOSTIC_WARNING, + LAZY_DIAGNOSTIC_ERROR, + LAZY_DIAGNOSTIC_FATAL +}; + +struct SC_LAZYFILE_EXPORT LazyDiagnostic { + LazyDiagnosticSeverity severity; + instanceID entity; + std::string type; + lazyFileOffset offset; + uint64_t line; + std::string attribute; + std::string message; + uint64_t occurrences; + + LazyDiagnostic(): severity( LAZY_DIAGNOSTIC_INFO ), entity( 0 ), offset( 0 ), + line( 0 ), occurrences( 1 ) {} +}; + +struct SC_LAZYFILE_EXPORT LazyScanProgress { + fileID file; + lazyFileOffset offset; + lazyFileOffset fileSize; + uint64_t instancesScanned; + + LazyScanProgress(): file( 0 ), offset( 0 ), fileSize( 0 ), instancesScanned( 0 ) {} +}; + +struct SC_LAZYFILE_EXPORT LazyCacheStatistics { + uint64_t instancesScanned; + uint64_t instancesLoaded; + uint64_t instancesPinned; + uint64_t cacheHighWater; + uint64_t cacheHits; + uint64_t cacheMisses; + uint64_t materializations; + uint64_t evictions; + uint64_t activeBatches; + uint64_t dataSections; + bool cancelled; + + LazyCacheStatistics(): instancesScanned( 0 ), instancesLoaded( 0 ), instancesPinned( 0 ), + cacheHighWater( 0 ), cacheHits( 0 ), cacheMisses( 0 ), materializations( 0 ), + evictions( 0 ), activeBatches( 0 ), dataSections( 0 ), cancelled( false ) {} +}; + +typedef std::function LazyProgressCallback; +typedef std::function LazyCancellationCallback; +typedef std::function LazyDiagnosticCallback; + +/** A non-owning, zero-copy view of an indexed instance-ID vector. + * The view remains valid until another file is scanned by the manager. + */ +class SC_LAZYFILE_EXPORT LazyInstanceIdView { + public: + typedef instanceRefs::const_iterator const_iterator; + + LazyInstanceIdView(): _ids( 0 ) {} + explicit LazyInstanceIdView( const instanceRefs * ids ): _ids( ids ) {} + + const_iterator begin() const; + const_iterator end() const; + size_t size() const; + bool empty() const; + instanceID operator[]( size_t index ) const; + + private: + const instanceRefs * _ids; +}; + +/** A dependency closure materialized and pinned for a bounded operation. + * STEPcode parsing remains single-threaded. Detach immutable conversion data + * before releasing the batch or passing work to another thread. + */ +class SC_LAZYFILE_EXPORT LazyInstanceBatch { + public: + LazyInstanceBatch(); + ~LazyInstanceBatch(); + LazyInstanceBatch( LazyInstanceBatch && other ); + LazyInstanceBatch & operator=( LazyInstanceBatch && other ); + + LazyInstanceBatch( const LazyInstanceBatch & ) = delete; + LazyInstanceBatch & operator=( const LazyInstanceBatch & ) = delete; + + bool valid() const; + void release(); + const std::vector & roots() const; + const std::vector & instances() const; + SDAI_Application_instance * get( instanceID id ) const; + + private: + friend class lazyInstMgr; + LazyInstanceBatch( lazyInstMgr * manager, const std::vector & roots, + const std::vector & instances ); + + lazyInstMgr * _manager; + std::vector _roots; + std::vector _instances; +}; + +/** Adapter preserving a stream-oriented diagnostic presentation. */ +class SC_LAZYFILE_EXPORT LazyTextDiagnosticAdapter { + public: + explicit LazyTextDiagnosticAdapter( std::ostream & stream ); + void operator()( const LazyDiagnostic & diagnostic ) const; + + private: + std::ostream * _stream; +}; + +#endif // LAZYSUPPORT_H diff --git a/include/cllazyfile/lazyTypes.h b/include/cllazyfile/lazyTypes.h index f1fdbd782..df8c7c631 100644 --- a/include/cllazyfile/lazyTypes.h +++ b/include/cllazyfile/lazyTypes.h @@ -6,6 +6,7 @@ #include #include #include +#include #ifdef HAVE_STDINT_H #include @@ -29,6 +30,7 @@ enum fileTypeEnum { Part21, Part28 }; // enum loadingEnum { immediate, lazy }; typedef uint64_t instanceID; ///< the number assigned to an instance in the file +typedef uint64_t lazyFileOffset; ///< byte offset in an exchange file typedef uint16_t sectionID; ///< globally unique index of a sectionReader in a sectionReaderVec_t typedef uint16_t fileID; ///< the index of a lazyFileReader in a lazyFileReaderVec_t. Can be inferred from a sectionID @@ -42,7 +44,10 @@ typedef uint16_t fileID; ///< the index of a lazyFileReader in a lazyFileRe * section = ( ps >> 48 ); * TODO: Also 8 bits of flags? Would allow files of 2^40 bytes, or ~1TB. */ -typedef uint64_t positionAndSection; +typedef struct { + lazyFileOffset begin; + sectionID section; +} instancePosition; typedef std::vector< instanceID > instanceRefs; @@ -55,7 +60,7 @@ typedef std::set< instanceID > instanceSet; * situations, so the information should be kept up-to-date. */ typedef struct { - long begin; ///< this is the result of tellg() before reading the instanceID; there may be whitespace or comments, but nothing else. + lazyFileOffset begin; ///< byte offset before the instance ID; whitespace or comments may precede it. instanceID instance; sectionID section; /* bool modified; */ /* this will be useful when writing instances - if an instance is @@ -67,6 +72,7 @@ typedef struct { lazyInstanceLoc loc; const char * name; instanceRefs * refs; + std::vector * componentTypes; ///< compositional types for a complex instance } namedLazyInstance; // instanceRefs - map between an instanceID and instances that refer to it @@ -80,7 +86,7 @@ typedef judyLArray< instanceID, SDAI_Application_instance * > instancesLoaded_t; // instanceStreamPos - map instance id to a streampos and data section // there could be multiple instances with the same ID, but in different files (or different sections of the same file?) -typedef judyL2Array< instanceID, positionAndSection > instanceStreamPos_t; +typedef judyL2Array< instanceID, instancePosition > instanceStreamPos_t; // data sections @@ -93,4 +99,3 @@ typedef std::vector< lazyFileReader * > lazyFileReaderVec_t; // NOTE not useful? typedef std::vector< lazyInstance > lazyInstanceVec_t; #endif //LAZYTYPES_H - diff --git a/include/cllazyfile/sectionReader.h b/include/cllazyfile/sectionReader.h index 74f1fa5bf..575fd96d3 100644 --- a/include/cllazyfile/sectionReader.h +++ b/include/cllazyfile/sectionReader.h @@ -51,7 +51,7 @@ class SC_LAZYFILE_EXPORT sectionReader { const char * getDelimitedKeyword( const char * delimiters ); /** Seek to the end of the current instance */ - std::streampos seekInstanceEnd( instanceRefs ** refs ); + std::streampos seekInstanceEnd( instanceRefs ** refs, std::vector * componentTypes = 0 ); /// operator>> is very slow?! inline void skipWS() { @@ -63,7 +63,7 @@ class SC_LAZYFILE_EXPORT sectionReader { STEPcomplex * CreateSubSuperInstance( const Registry * reg, instanceID fileid, Severity & sev ); public: - SDAI_Application_instance * getRealInstance( const Registry * reg, long int begin, instanceID instance, + SDAI_Application_instance * getRealInstance( const Registry * reg, lazyFileOffset begin, instanceID instance, const std::string & typeName = "", const std::string & schName = "", bool header = false ); sectionID ID() const { @@ -91,11 +91,11 @@ class SC_LAZYFILE_EXPORT sectionReader { /** returns the type string for an instance, read straight from the file * if this function changes, probably need to change nextInstance() as well * don't check errors - they would have been encountered during the initial file scan, and the file is still open so it can't have been modified */ - const char * getType( long int offset ) { + const char * getType( lazyFileOffset offset ) { if( offset <= 0 ) { return 0; } - _file.seekg( offset ); + _file.seekg( static_cast( offset ) ); readInstanceNumber(); skipWS(); return getDelimitedKeyword( ";( /\\" ); @@ -112,4 +112,3 @@ class SC_LAZYFILE_EXPORT sectionReader { }; #endif //SECTIONREADER_H - diff --git a/src/cllazyfile/CMakeLists.txt b/src/cllazyfile/CMakeLists.txt index 83b2d7137..89b84ee06 100644 --- a/src/cllazyfile/CMakeLists.txt +++ b/src/cllazyfile/CMakeLists.txt @@ -2,6 +2,7 @@ set(LAZY_SRCS lazyDataSectionReader.cc lazyFileReader.cc lazyInstMgr.cc + lazySupport.cc p21HeaderSectionReader.cc sectionReader.cc lazyP21DataSectionReader.cc @@ -42,3 +43,15 @@ endif() # End: # ex: shiftwidth=2 tabstop=8 +if(SC_ENABLE_TESTING) + if(BUILD_SHARED_LIBS) + set(_lazy_index_libs steplazyfile stepeditor stepcore stepdai steputils) + else() + set(_lazy_index_libs steplazyfile-static stepeditor-static + stepcore-static stepdai-static steputils-static) + endif() + SC_ADDEXEC(lazy_index_test SOURCES lazy_index_test.cc + LINK_LIBRARIES ${_lazy_index_libs} NO_INSTALL) + add_test(NAME lazy_index + COMMAND lazy_index_test ${CMAKE_CURRENT_SOURCE_DIR}/test/lazy_index.stp) +endif() diff --git a/src/cllazyfile/lazyDataSectionReader.cc b/src/cllazyfile/lazyDataSectionReader.cc index 6de006fe4..3fbb802f2 100644 --- a/src/cllazyfile/lazyDataSectionReader.cc +++ b/src/cllazyfile/lazyDataSectionReader.cc @@ -6,7 +6,7 @@ lazyDataSectionReader::lazyDataSectionReader( lazyFileReader * parent, std::ifstream & file, std::streampos start, sectionID sid ): sectionReader( parent, file, start, sid ) { - _sectionIdentifier = ""; //FIXME set _sectionIdentifier from the data section identifier (2002 rev of Part 21), if present + _sectionIdentifier = ""; //FIXME retain the data section identifier (2002 revision of Part 21) _error = false; + _completelyLoaded = false; } - diff --git a/src/cllazyfile/lazyFileReader.cc b/src/cllazyfile/lazyFileReader.cc index fed35c6f5..3705b5517 100644 --- a/src/cllazyfile/lazyFileReader.cc +++ b/src/cllazyfile/lazyFileReader.cc @@ -19,20 +19,44 @@ void lazyFileReader::initP21() { } _parent->registerDataSection( r ); + if( _parent->cancelled() ) break; + //check for new data section (DATA) or end of file (END-ISO-10303-21;) - while( isspace( _file.peek() ) && _file.good() ) { - _file.ignore( 1 ); + for( ;; ) { + while( isspace( _file.peek() ) && _file.good() ) _file.ignore( 1 ); + if( _file.peek() != '/' ) break; + std::streampos comment = _file.tellg(); + _file.get(); + if( _file.peek() != '*' ) { + _file.seekg( comment ); + break; + } + _file.get(); + int previous = 0; + int current = 0; + while( _file.good() ) { + current = _file.get(); + if( previous == '*' && current == '/' ) break; + previous = current; + } } + std::streampos nextSection = _file.tellg(); if( needKW( "END-ISO-10303-21;" ) ) { break; - } else if( !needKW( "DATA" ) ) { + } + _file.clear(); + _file.seekg( nextSection ); + if( !needKW( "DATA" ) ) { std::cerr << "Corrupted file - did not find new data section (\"DATA\") or end of file (\"END-ISO-10303-21;\") at offset " << _file.tellg() << std::endl; break; } + _file.clear(); + _file.seekg( nextSection ); } } bool lazyFileReader::needKW( const char * kw ) { + std::streampos start = _file.tellg(); const char * c = kw; bool found = true; while( *c ) { @@ -42,6 +66,10 @@ bool lazyFileReader::needKW( const char * kw ) { } c++; } + if( !found ) { + _file.clear(); + _file.seekg( start ); + } return found; } @@ -49,11 +77,25 @@ instancesLoaded_t * lazyFileReader::getHeaderInstances() { return _header->getInstances(); } -lazyFileReader::lazyFileReader( std::string fname, lazyInstMgr * i, fileID fid ): _fileName( fname ), _parent( i ), _fileID( fid ) { +lazyFileReader::lazyFileReader( std::string fname, lazyInstMgr * i, fileID fid ): _fileName( fname ), _parent( i ), + _header( 0 ), _fileID( fid ), _fileSize( 0 ), _valid( false ) { _file.open( _fileName.c_str(), std::ios::binary ); _file.imbue( std::locale::classic() ); _file.unsetf( std::ios_base::skipws ); - assert( _file.is_open() && _file.good() ); + if( !_file.is_open() || !_file.good() ) { + LazyDiagnostic diagnostic; + diagnostic.severity = LAZY_DIAGNOSTIC_FATAL; + diagnostic.message = "unable to open exchange file: " + _fileName; + _parent->emitDiagnostic( diagnostic ); + return; + } + + _file.seekg( 0, std::ios::end ); + std::streampos end = _file.tellg(); + if( end != std::streampos( -1 ) ) { + _fileSize = static_cast( static_cast( end ) ); + } + _file.seekg( 0, std::ios::beg ); detectType(); switch( _fileType ) { @@ -67,9 +109,9 @@ lazyFileReader::lazyFileReader( std::string fname, lazyInstMgr * i, fileID fid ) std::cerr << "Reached default case, " << __FILE__ << ":" << __LINE__ << std::endl; abort(); } + _valid = true; } lazyFileReader::~lazyFileReader() { delete _header; } - diff --git a/src/cllazyfile/lazyInstMgr.cc b/src/cllazyfile/lazyInstMgr.cc index 65f3e7558..20857ecd1 100644 --- a/src/cllazyfile/lazyInstMgr.cc +++ b/src/cllazyfile/lazyInstMgr.cc @@ -8,22 +8,33 @@ #include "clstepcore/sdaiApplication_instance.h" +#include +#include +#include + lazyInstMgr::lazyInstMgr() { _headerRegistry = new Registry( HeaderSchemaInit ); _instanceTypes = new instanceTypes_t( 255 ); //NOTE arbitrary max of 255 chars for a type name _lazyInstanceCount = 0; _loadedInstanceCount = 0; + _cacheHighWater = 0; + _cacheHits = 0; + _cacheMisses = 0; + _materializations = 0; + _evictions = 0; + _activeBatches = 0; + _batchLoadDepth = 0; + _cancelled = false; _longestTypeNameLen = 0; _mainRegistry = 0; + _ownsMainRegistry = false; _errors = new ErrorDescriptor(); _ima = new instMgrAdapter( this ); } lazyInstMgr::~lazyInstMgr() { - delete _headerRegistry; - delete _errors; - delete _ima; - //loop over files, sections, instances; delete header instances + // Keep registries and the adapter alive until their instances are gone. + _instancesLoaded.clear( true ); lazyFileReaderVec_t::iterator fit = _files.begin(); for( ; fit != _files.end(); ++fit ) { delete *fit; @@ -32,8 +43,12 @@ lazyInstMgr::~lazyInstMgr() { for( ; sit != _dataSections.end(); ++sit ) { delete *sit; } - _instancesLoaded.clear(); _instanceStreamPos.clear(); + delete _instanceTypes; + delete _ima; + if( _ownsMainRegistry ) delete _mainRegistry; + delete _headerRegistry; + delete _errors; } sectionID lazyInstMgr::registerDataSection( lazyDataSectionReader * sreader ) { @@ -50,18 +65,32 @@ void lazyInstMgr::addLazyInstance( namedLazyInstance inst ) { _longestTypeName = inst.name; } _instanceTypes->insert( inst.name, inst.loc.instance ); - /* store 16 bits of section id and 48 of instance offset into one 64-bit int - ** TODO: check and warn if anything is lost (in calling code?) - ** does 32bit need anything special? - ** - ** create conversion class? - ** could then initialize conversion object with number of bits - ** also a good place to check for data loss - */ - positionAndSection ps = inst.loc.section; - ps <<= 48; - ps |= ( inst.loc.begin & 0xFFFFFFFFFFFFULL ); - _instanceStreamPos.insert( inst.loc.instance, ps ); + if( inst.componentTypes ) { + std::vector::const_iterator type = inst.componentTypes->begin(); + for( ; type != inst.componentTypes->end(); ++type ) { + _instanceTypes->insert( type->c_str(), inst.loc.instance ); + if( static_cast( type->size() ) > _longestTypeNameLen ) { + _longestTypeNameLen = type->size(); + _longestTypeName = *type; + } + } + delete inst.componentTypes; + } + const bool duplicate = _instanceStreamPos.find( inst.loc.instance ) != 0; + if( !duplicate ) _allInstances.push_back( inst.loc.instance ); + instancePosition pos; + pos.begin = inst.loc.begin; + pos.section = inst.loc.section; + if( duplicate ) { + LazyDiagnostic diagnostic; + diagnostic.severity = LAZY_DIAGNOSTIC_WARNING; + diagnostic.entity = inst.loc.instance; + diagnostic.type = inst.name; + diagnostic.offset = inst.loc.begin; + diagnostic.message = "duplicate instance identifier"; + emitDiagnostic( diagnostic ); + } + _instanceStreamPos.insert( inst.loc.instance, pos ); if( inst.refs ) { if( inst.refs->size() > 0 ) { @@ -72,10 +101,80 @@ void lazyInstMgr::addLazyInstance( namedLazyInstance inst ) { //reverse refs _revInstanceRefs.insert( *it, inst.loc.instance ); } - } else { - delete inst.refs; + } + delete inst.refs; + } +} + +LazyInstanceIdView lazyInstMgr::instancesByType( std::string type, bool caseSensitive ) { + if( !caseSensitive ) { + std::string::iterator it = type.begin(); + for( ; it != type.end(); ++it ) { + *it = static_cast( toupper( static_cast( *it ) ) ); } } + return LazyInstanceIdView( _instanceTypes->find( type.c_str() ) ); +} + +LazyInstanceIdView lazyInstMgr::forwardReferences( instanceID id ) { + return LazyInstanceIdView( _fwdInstanceRefs.find( id ) ); +} + +LazyInstanceIdView lazyInstMgr::reverseReferences( instanceID id ) { + return LazyInstanceIdView( _revInstanceRefs.find( id ) ); +} + +void lazyInstMgr::observeScan( fileID file, lazyFileOffset offset, lazyFileOffset fileSize ) { + if( _cancelled ) return; + if( _cancellationCallback && _cancellationCallback() ) { + _cancelled = true; + LazyDiagnostic diagnostic; + diagnostic.severity = LAZY_DIAGNOSTIC_INFO; + diagnostic.offset = offset; + diagnostic.message = "scan cancelled"; + emitDiagnostic( diagnostic ); + return; + } + if( _progressCallback && ( _lazyInstanceCount == 1 || ( _lazyInstanceCount % 4096 ) == 0 || offset >= fileSize ) ) { + LazyScanProgress progress; + progress.file = file; + progress.offset = offset; + progress.fileSize = fileSize; + progress.instancesScanned = _lazyInstanceCount; + _progressCallback( progress ); + } +} + +void lazyInstMgr::emitDiagnostic( LazyDiagnostic diagnostic ) { + std::ostringstream key; + key << diagnostic.type << '#' << diagnostic.entity << ':' << diagnostic.attribute << ':' << diagnostic.message; + uint64_t & count = _diagnosticCounts[key.str()]; + ++count; + diagnostic.occurrences = count; + if( _diagnosticCallback && count == 1 ) { + _diagnosticCallback( diagnostic ); + } +} + +uint64_t lazyInstMgr::diagnosticCount( const std::string & key ) const { + std::map::const_iterator found = _diagnosticCounts.find( key ); + return found == _diagnosticCounts.end() ? 0 : found->second; +} + +LazyCacheStatistics lazyInstMgr::cacheStatistics() const { + LazyCacheStatistics stats; + stats.instancesScanned = _lazyInstanceCount; + stats.instancesLoaded = _loadedInstanceCount; + stats.instancesPinned = _pinCounts.size(); + stats.cacheHighWater = _cacheHighWater; + stats.cacheHits = _cacheHits; + stats.cacheMisses = _cacheMisses; + stats.materializations = _materializations; + stats.evictions = _evictions; + stats.activeBatches = _activeBatches; + stats.dataSections = _dataSections.size(); + stats.cancelled = _cancelled; + return stats; } unsigned long lazyInstMgr::getNumTypes() const { @@ -93,7 +192,7 @@ unsigned long lazyInstMgr::getNumTypes() const { return n ; } -void lazyInstMgr::openFile( std::string fname ) { +bool lazyInstMgr::openFile( std::string fname ) { //don't want to hold a lock for the entire time we're reading the file. //create a place in the vector and remember its location, then free lock ///FIXME begin atomic op @@ -101,20 +200,60 @@ void lazyInstMgr::openFile( std::string fname ) { _files.push_back( (lazyFileReader * ) 0 ); ///FIXME end atomic op lazyFileReader * lfr = new lazyFileReader( fname, this, i ); + if( !lfr->valid() ) { + delete lfr; + _files.pop_back(); + return false; + } _files[i] = lfr; + validateReferences(); /// TODO resolve inverse attr references //between instances, or eDesc --> inst???? + return true; +} + +void lazyInstMgr::validateReferences() { + instanceRefs_t::cpair current = _fwdInstanceRefs.begin(); + while( current.value ) { + instanceRefs_t::cvector::const_iterator ref = current.value->begin(); + for( ; ref != current.value->end(); ++ref ) { + if( !_instanceStreamPos.find( *ref ) ) { + LazyDiagnostic diagnostic; + diagnostic.severity = LAZY_DIAGNOSTIC_ERROR; + diagnostic.entity = current.key; + const char * type = typeFromFile( current.key ); + if( type ) diagnostic.type = type; + instanceStreamPos_t::cvector * positions = _instanceStreamPos.find( current.key ); + if( positions && !positions->empty() ) diagnostic.offset = positions->front().begin; + std::ostringstream message; + message << "reference to missing instance #" << *ref; + diagnostic.message = message.str(); + emitDiagnostic( diagnostic ); + } + } + current = _fwdInstanceRefs.next(); + } } SDAI_Application_instance * lazyInstMgr::loadInstance( instanceID id, bool reSeek ) { assert( _mainRegistry && "Main registry has not been initialized. Do so with initRegistry() or setRegistry()." ); std::streampos oldPos; - positionAndSection ps; - sectionID sid; + instancePosition pos; SDAI_Application_instance * inst = _instancesLoaded.find( id ); if( inst ) { + ++_cacheHits; + if( _batchLoadDepth == 0 ) _permanentlyLoadedInstances.insert( id ); return inst; } + ++_cacheMisses; + if( id > static_cast( INT_MAX ) ) { + LazyDiagnostic diagnostic; + diagnostic.severity = LAZY_DIAGNOSTIC_ERROR; + diagnostic.entity = id; + diagnostic.message = "instance ID exceeds the current SDAI materialization limit"; + emitDiagnostic( diagnostic ); + return 0; + } instanceStreamPos_t::cvector * cv; if( 0 != ( cv = _instanceStreamPos.find( id ) ) ) { switch( cv->size() ) { @@ -122,37 +261,130 @@ SDAI_Application_instance * lazyInstMgr::loadInstance( instanceID id, bool reSee std::cerr << "Instance #" << id << " not found in any section." << std::endl; break; case 1: - long int off; - ps = cv->at( 0 ); - off = ps & 0xFFFFFFFFFFFFULL; - sid = ps >> 48; - assert( _dataSections.size() > sid ); + pos = cv->at( 0 ); + assert( _dataSections.size() > pos.section ); if( reSeek ) { - oldPos = _dataSections[sid]->tellg(); + oldPos = _dataSections[pos.section]->tellg(); } - inst = _dataSections[sid]->getRealInstance( _mainRegistry, off, id ); + inst = _dataSections[pos.section]->getRealInstance( _mainRegistry, pos.begin, id ); if( reSeek ) { - _dataSections[sid]->seekg( oldPos ); + _dataSections[pos.section]->seekg( oldPos ); } break; default: std::cerr << "Instance #" << id << " exists in multiple sections. This is not yet supported." << std::endl; + { + LazyDiagnostic diagnostic; + diagnostic.severity = LAZY_DIAGNOSTIC_ERROR; + diagnostic.entity = id; + diagnostic.message = "instance identifier occurs in multiple DATA sections"; + emitDiagnostic( diagnostic ); + } break; } if( !isNilSTEPentity( inst ) ) { _instancesLoaded.insert( id, inst ); _loadedInstanceCount++; - lazyRefs lr( this, inst ); - lazyRefs::referentInstances_t insts = lr.result(); + ++_materializations; + _cacheHighWater = std::max( _cacheHighWater, _loadedInstanceCount ); + if( _batchLoadDepth ) { + _batchOwnedInstances.insert( id ); + } else { + _permanentlyLoadedInstances.insert( id ); + lazyRefs lr( this, inst ); + lazyRefs::referentInstances_t insts = lr.result(); + } } else { std::cerr << "Error loading instance #" << id << "." << std::endl; + LazyDiagnostic diagnostic; + diagnostic.severity = LAZY_DIAGNOSTIC_ERROR; + diagnostic.entity = id; + diagnostic.offset = pos.begin; + diagnostic.message = "SDAI instance materialization failed"; + emitDiagnostic( diagnostic ); } } else { std::cerr << "Instance #" << id << " not found in any section." << std::endl; + LazyDiagnostic diagnostic; + diagnostic.severity = LAZY_DIAGNOSTIC_ERROR; + diagnostic.entity = id; + diagnostic.message = "instance not found in any DATA section"; + emitDiagnostic( diagnostic ); } return inst; } +SDAI_Application_instance * lazyInstMgr::cachedInstance( instanceID id ) { + return _instancesLoaded.find( id ); +} + +std::vector lazyInstMgr::dependencyClosure( const std::vector & roots ) { + std::set closure( roots.begin(), roots.end() ); + std::vector queue( roots.begin(), roots.end() ); + size_t current = 0; + while( current < queue.size() ) { + instanceRefs_t::cvector * refs = _fwdInstanceRefs.find( queue[current++] ); + if( !refs ) continue; + instanceRefs_t::cvector::const_iterator ref = refs->begin(); + for( ; ref != refs->end(); ++ref ) { + if( closure.insert( *ref ).second ) queue.push_back( *ref ); + } + } + return std::vector( closure.begin(), closure.end() ); +} + +LazyInstanceBatch lazyInstMgr::loadBatch( instanceID root ) { + std::vector roots( 1, root ); + return loadBatch( roots ); +} + +LazyInstanceBatch lazyInstMgr::loadBatch( const std::vector & roots ) { + std::vector closure = dependencyClosure( roots ); + std::vector::const_iterator id = closure.begin(); + for( ; id != closure.end(); ++id ) ++_pinCounts[*id]; + ++_activeBatches; + ++_batchLoadDepth; + for( id = closure.begin(); id != closure.end(); ++id ) { + if( !loadInstance( *id, true ) ) { + LazyDiagnostic diagnostic; + diagnostic.severity = LAZY_DIAGNOSTIC_ERROR; + diagnostic.entity = *id; + diagnostic.message = "dependency batch could not materialize instance"; + emitDiagnostic( diagnostic ); + } + } + --_batchLoadDepth; + return LazyInstanceBatch( this, roots, closure ); +} + +void lazyInstMgr::releaseBatch( const std::vector & instances ) { + std::vector evict; + std::vector::const_iterator id = instances.begin(); + for( ; id != instances.end(); ++id ) { + std::map::iterator pin = _pinCounts.find( *id ); + if( pin == _pinCounts.end() ) continue; + if( --pin->second == 0 ) { + _pinCounts.erase( pin ); + if( _batchOwnedInstances.count( *id ) && !_permanentlyLoadedInstances.count( *id ) ) { + evict.push_back( *id ); + } + } + } + std::vector deleted; + for( id = evict.begin(); id != evict.end(); ++id ) { + SDAI_Application_instance * inst = _instancesLoaded.find( *id ); + if( !inst ) continue; + _instancesLoaded.removeEntry( *id ); + _batchOwnedInstances.erase( *id ); + deleted.push_back( inst ); + --_loadedInstanceCount; + ++_evictions; + } + std::vector::iterator inst = deleted.begin(); + for( ; inst != deleted.end(); ++inst ) delete *inst; + if( _activeBatches ) --_activeBatches; +} + instanceSet * lazyInstMgr::instanceDependencies( instanceID id ) { instanceSet * checkedDependencies = new instanceSet(); @@ -182,4 +414,3 @@ instanceSet * lazyInstMgr::instanceDependencies( instanceID id ) { return checkedDependencies; } - diff --git a/src/cllazyfile/lazyP21DataSectionReader.cc b/src/cllazyfile/lazyP21DataSectionReader.cc index 0716f5c5f..b768c6138 100644 --- a/src/cllazyfile/lazyP21DataSectionReader.cc +++ b/src/cllazyfile/lazyP21DataSectionReader.cc @@ -3,13 +3,60 @@ #include "cllazyfile/lazyP21DataSectionReader.h" #include "cllazyfile/lazyInstMgr.h" +void lazyP21DataSectionReader::findSectionStart() { + std::streampos keywordEnd = findNormalString( "DATA" ); + if( keywordEnd == std::streampos( -1 ) ) { + _sectionStart = -1; + return; + } + _file.seekg( keywordEnd ); + skipWS(); + int depth = 0; + int current = _file.get(); + if( current == ';' ) { + _sectionStart = _file.tellg(); + return; + } + if( current != '(' ) { + _sectionStart = -1; + return; + } + depth = 1; + while( depth && _file.good() ) { + current = _file.get(); + if( current == '\'' ) { + _file.seekg( _file.tellg() - std::streampos( 1 ) ); + GetLiteralStr( _file, _lazyFile->getInstMgr()->getErrorDesc() ); + } else if( current == '/' && _file.peek() == '*' ) { + findNormalString( "*/" ); + } else if( current == '(' ) { + ++depth; + } else if( current == ')' ) { + --depth; + } + } + skipWS(); + if( depth == 0 && _file.get() == ';' ) { + _sectionStart = _file.tellg(); + } else { + _sectionStart = -1; + } +} + lazyP21DataSectionReader::lazyP21DataSectionReader( lazyFileReader * parent, std::ifstream & file, std::streampos start, sectionID sid ): lazyDataSectionReader( parent, file, start, sid ) { findSectionStart(); + if( _sectionStart == std::streampos( -1 ) ) { + _error = true; + return; + } namedLazyInstance nl; while( nl = nextInstance(), ( ( nl.loc.begin > 0 ) && ( nl.name != 0 ) ) ) { parent->getInstMgr()->addLazyInstance( nl ); + parent->getInstMgr()->observeScan( parent->ID(), + static_cast( static_cast( _file.tellg() ) ), parent->fileSize() ); + if( parent->getInstMgr()->cancelled() ) return; } if( sectionReader::_error->severity() <= SEVERITY_WARNING ) { @@ -49,26 +96,33 @@ const namedLazyInstance lazyP21DataSectionReader::nextInstance() { namedLazyInstance i; i.refs = 0; + i.componentTypes = 0; i.loc.section = 0; - i.loc.begin = _file.tellg(); + std::streampos start = _file.tellg(); + i.loc.begin = start == std::streampos( -1 ) ? 0 : + static_cast( static_cast( start ) ); i.loc.instance = readInstanceNumber(); if( ( _file.good() ) && ( i.loc.instance > 0 ) ) { skipWS(); i.loc.section = _sectionID; i.name = getDelimitedKeyword( ";( /\\" ); if( _file.good() ) { - end = seekInstanceEnd( & i.refs ); + if( i.name[0] == '\0' ) i.componentTypes = new std::vector; + end = seekInstanceEnd( & i.refs, i.componentTypes ); } } if( ( i.loc.instance == 0 ) || ( !_file.good() ) || ( end == ( std::streampos ) - 1 ) ) { //invalid instance, so clear everything _file.seekg( i.loc.begin ); - i.loc.begin = -1; + i.loc.begin = 0; if( i.refs ) { delete i.refs; } + if( i.componentTypes ) { + delete i.componentTypes; + i.componentTypes = 0; + } i.name = 0; } return i; } - diff --git a/src/cllazyfile/lazySupport.cc b/src/cllazyfile/lazySupport.cc new file mode 100644 index 000000000..ebebe7b0e --- /dev/null +++ b/src/cllazyfile/lazySupport.cc @@ -0,0 +1,106 @@ +#include +#include +#include + +#include "cllazyfile/lazySupport.h" +#include "cllazyfile/lazyInstMgr.h" + +namespace { +const instanceRefs & emptyIds() { + static const instanceRefs ids; + return ids; +} +} + +LazyInstanceIdView::const_iterator LazyInstanceIdView::begin() const { + return _ids ? _ids->begin() : emptyIds().begin(); +} + +LazyInstanceIdView::const_iterator LazyInstanceIdView::end() const { + return _ids ? _ids->end() : emptyIds().end(); +} + +size_t LazyInstanceIdView::size() const { + return _ids ? _ids->size() : 0; +} + +bool LazyInstanceIdView::empty() const { + return size() == 0; +} + +instanceID LazyInstanceIdView::operator[]( size_t index ) const { + if( !_ids || index >= _ids->size() ) { + throw std::out_of_range( "LazyInstanceIdView index" ); + } + return ( *_ids )[index]; +} + +LazyInstanceBatch::LazyInstanceBatch(): _manager( 0 ) {} + +LazyInstanceBatch::LazyInstanceBatch( lazyInstMgr * manager, + const std::vector & roots, const std::vector & instances ): + _manager( manager ), _roots( roots ), _instances( instances ) {} + +LazyInstanceBatch::~LazyInstanceBatch() { + release(); +} + +LazyInstanceBatch::LazyInstanceBatch( LazyInstanceBatch && other ): + _manager( other._manager ), _roots( std::move( other._roots ) ), + _instances( std::move( other._instances ) ) { + other._manager = 0; +} + +LazyInstanceBatch & LazyInstanceBatch::operator=( LazyInstanceBatch && other ) { + if( this != &other ) { + release(); + _manager = other._manager; + _roots = std::move( other._roots ); + _instances = std::move( other._instances ); + other._manager = 0; + } + return *this; +} + +bool LazyInstanceBatch::valid() const { + return _manager != 0; +} + +void LazyInstanceBatch::release() { + if( _manager ) { + _manager->releaseBatch( _instances ); + _manager = 0; + } + _roots.clear(); + _instances.clear(); +} + +const std::vector & LazyInstanceBatch::roots() const { + return _roots; +} + +const std::vector & LazyInstanceBatch::instances() const { + return _instances; +} + +SDAI_Application_instance * LazyInstanceBatch::get( instanceID id ) const { + return _manager ? _manager->cachedInstance( id ) : 0; +} + +LazyTextDiagnosticAdapter::LazyTextDiagnosticAdapter( std::ostream & stream ): _stream( &stream ) {} + +void LazyTextDiagnosticAdapter::operator()( const LazyDiagnostic & diagnostic ) const { + const char * label = "info"; + if( diagnostic.severity == LAZY_DIAGNOSTIC_WARNING ) label = "warning"; + if( diagnostic.severity == LAZY_DIAGNOSTIC_ERROR ) label = "error"; + if( diagnostic.severity == LAZY_DIAGNOSTIC_FATAL ) label = "fatal"; + ( *_stream ) << label; + if( diagnostic.entity ) ( *_stream ) << " #" << diagnostic.entity; + if( !diagnostic.type.empty() ) ( *_stream ) << " " << diagnostic.type; + if( diagnostic.offset ) ( *_stream ) << " at offset " << diagnostic.offset; + if( diagnostic.line ) ( *_stream ) << " line " << diagnostic.line; + if( !diagnostic.attribute.empty() ) ( *_stream ) << " attribute " << diagnostic.attribute; + ( *_stream ) << ": " << diagnostic.message; + if( diagnostic.occurrences > 1 ) ( *_stream ) << " (" << diagnostic.occurrences << " occurrences)"; + ( *_stream ) << std::endl; +} diff --git a/src/cllazyfile/lazy_index_test.cc b/src/cllazyfile/lazy_index_test.cc new file mode 100644 index 000000000..ce59013b8 --- /dev/null +++ b/src/cllazyfile/lazy_index_test.cc @@ -0,0 +1,60 @@ +#include +#include + +#include "cllazyfile/lazyInstMgr.h" + +namespace { +void require( bool condition, const char * message ) { + if( !condition ) { + std::cerr << "lazy_index_test: " << message << std::endl; + std::exit( EXIT_FAILURE ); + } +} +} + +int main( int argc, char ** argv ) { + require( argc == 2, "expected fixture path" ); + + lazyInstMgr manager; + uint64_t progressCalls = 0; + uint64_t diagnosticCalls = 0; + LazyDiagnostic missingReference; + manager.setProgressCallback( [&progressCalls]( const LazyScanProgress & progress ) { + require( progress.fileSize >= progress.offset, "progress offset exceeds file size" ); + ++progressCalls; + } ); + manager.setDiagnosticCallback( [&diagnosticCalls, &missingReference]( const LazyDiagnostic & diagnostic ) { + ++diagnosticCalls; + if( diagnostic.entity == 5 ) missingReference = diagnostic; + } ); + manager.openFile( argv[1] ); + + LazyCacheStatistics stats = manager.cacheStatistics(); + require( stats.instancesScanned == 5, "wrong scan count" ); + require( stats.dataSections == 2, "second DATA section was not indexed" ); + require( progressCalls > 0, "progress callback was not called" ); + require( manager.instancesByType( "a" ).size() == 1, "case-insensitive type index failed" ); + require( manager.allInstances().size() == manager.totalInstanceCount(), "all-instance index count failed" ); + require( manager.instancesByType( "C" ).size() == 1, "first complex component not indexed" ); + require( manager.instancesByType( "D" ).size() == 1, "second complex component not indexed" ); + require( manager.instancesByType( "" ).size() == 1, "complex instance index failed" ); + require( diagnosticCalls == 1, "structured diagnostic callback was not bounded" ); + require( missingReference.severity == LAZY_DIAGNOSTIC_ERROR && missingReference.offset > 0, + "missing-reference diagnostic lacks structured context" ); + require( missingReference.message == "reference to missing instance #99", "wrong missing-reference diagnostic" ); + require( manager.forwardReferences( 1 ).size() == 1 && manager.forwardReferences( 1 )[0] == 2, + "forward reference index failed" ); + require( manager.reverseReferences( 1 ).size() == 1 && manager.reverseReferences( 1 )[0] == 3, + "reverse reference index failed" ); + + lazyInstMgr cancelled; + uint64_t cancellationCalls = 0; + cancelled.setCancellationCallback( [&cancellationCalls]() { + return ++cancellationCalls >= 2; + } ); + cancelled.openFile( argv[1] ); + stats = cancelled.cacheStatistics(); + require( stats.cancelled, "cancellation was not recorded" ); + require( stats.instancesScanned == 2, "scan did not stop at cancellation boundary" ); + return EXIT_SUCCESS; +} diff --git a/src/cllazyfile/lazy_test.cc b/src/cllazyfile/lazy_test.cc index d5ce6081a..246f7429f 100644 --- a/src/cllazyfile/lazy_test.cc +++ b/src/cllazyfile/lazy_test.cc @@ -171,6 +171,34 @@ int main( int argc, char ** argv ) { #ifndef NO_REGISTRY if( instWithRef ) { std::cout << "Number of data section instances fully loaded: " << mgr->loadedInstanceCount() << std::endl; + { + LazyInstanceBatch first = mgr->loadBatch( instWithRef ); + LazyInstanceBatch second = mgr->loadBatch( instWithRef ); + size_t sharedSize = second.instances().size(); + first.release(); + if( mgr->loadedInstanceCount() != sharedSize || !second.get( instWithRef ) ) { + std::cerr << "Releasing one batch evicted instances pinned by another batch" << std::endl; + return EXIT_FAILURE; + } + } + if( mgr->loadedInstanceCount() != 0 ) { + std::cerr << "Shared dependency batches did not release their cache" << std::endl; + return EXIT_FAILURE; + } + for( int cycle = 0; cycle < 3; ++cycle ) { + { + LazyInstanceBatch batch = mgr->loadBatch( instWithRef ); + if( !batch.valid() || !batch.get( instWithRef ) ) { + std::cerr << "Unable to materialize dependency batch for #" << instWithRef << std::endl; + return EXIT_FAILURE; + } + std::cout << "Batch-pinned instances: " << batch.instances().size() << std::endl; + } + if( mgr->loadedInstanceCount() != 0 ) { + std::cerr << "Dependency batch did not return the cache to its initial size" << std::endl; + return EXIT_FAILURE; + } + } std::cout << "Loading #" << instWithRef; SDAI_Application_instance * inst = mgr->loadInstance( instWithRef ); std::cout << " which is of type " << inst->EntityName() << std::endl; @@ -193,4 +221,3 @@ int main( int argc, char ** argv ) { delete mgr; //stats will print from its destructor } - diff --git a/src/cllazyfile/p21HeaderSectionReader.cc b/src/cllazyfile/p21HeaderSectionReader.cc index 037604fc4..e11f1b14f 100644 --- a/src/cllazyfile/p21HeaderSectionReader.cc +++ b/src/cllazyfile/p21HeaderSectionReader.cc @@ -34,7 +34,10 @@ const namedLazyInstance p21HeaderSectionReader::nextInstance() { static instanceID nextFreeInstance = 4; // 1-3 are reserved per 10303-21 i.refs = 0; - i.loc.begin = _file.tellg(); + i.componentTypes = 0; + std::streampos start = _file.tellg(); + i.loc.begin = start == std::streampos( -1 ) ? 0 : + static_cast( static_cast( start ) ); i.loc.section = _sectionID; skipWS(); if( i.loc.begin <= 0 ) { @@ -57,10 +60,9 @@ const namedLazyInstance p21HeaderSectionReader::nextInstance() { std::streampos end = seekInstanceEnd( 0 ); //no references in file header if( ( (signed long int)end == -1 ) || ( end >= _sectionEnd ) ) { //invalid instance, so clear everything - i.loc.begin = -1; + i.loc.begin = 0; i.name = 0; } } return i; } - diff --git a/src/cllazyfile/sectionReader.cc b/src/cllazyfile/sectionReader.cc index 4a159439a..ef4a5c427 100644 --- a/src/cllazyfile/sectionReader.cc +++ b/src/cllazyfile/sectionReader.cc @@ -115,7 +115,7 @@ const char * sectionReader::getDelimitedKeyword( const char * delimiters ) { } } c = _file.peek(); - if( !strchr( delimiters, c ) ) { + if( !strchr( delimiters, c ) && !isspace( static_cast( c ) ) ) { std::cerr << SC_CURRENT_FUNCTION << ": missing delimiter. Found " << c << ", expected one of " << delimiters << " at end of keyword " << str << ". File offset: " << _file.tellg() << std::endl; abort(); } @@ -125,13 +125,15 @@ const char * sectionReader::getDelimitedKeyword( const char * delimiters ) { /// search forward in the file for the end of the instance. Start position should /// be the opening parenthesis; otherwise, it is likely to fail. ///NOTE *must* check return value! -std::streampos sectionReader::seekInstanceEnd( instanceRefs ** refs ) { +std::streampos sectionReader::seekInstanceEnd( instanceRefs ** refs, std::vector * componentTypes ) { int c; int parenDepth = 0; + bool expectComplexType = false; while( c = _file.get(), _file.good() ) { switch( c ) { case '(': parenDepth++; + if( componentTypes && parenDepth == 1 ) expectComplexType = true; break; case '/': if( _file.peek() == '*' ) { @@ -162,7 +164,8 @@ std::streampos sectionReader::seekInstanceEnd( instanceRefs ** refs ) { } break; case ')': - if( --parenDepth == 0 ) { + if( --parenDepth == 1 && componentTypes ) expectComplexType = true; + if( parenDepth == 0 ) { skipWS(); if( _file.get() == ';' ) { return _file.tellg(); @@ -170,7 +173,23 @@ std::streampos sectionReader::seekInstanceEnd( instanceRefs ** refs ) { _file.seekg( _file.tellg() - std::streampos(1) ); } } + break; default: + if( componentTypes && parenDepth == 1 && expectComplexType && + ( isupper( c ) || c == '!' ) ) { + std::string type( 1, static_cast( c ) ); + while( _file.good() ) { + int next = _file.get(); + if( next == '-' || next == '_' || isupper( next ) || isdigit( next ) ) { + type.push_back( static_cast( next ) ); + } else { + _file.putback( static_cast( next ) ); + break; + } + } + componentTypes->push_back( type ); + expectComplexType = false; + } break; } } @@ -261,10 +280,11 @@ instanceID sectionReader::readInstanceNumber() { /** load an instance and return a pointer to it. * side effect: recursively loads any instances the specified instance depends upon */ -SDAI_Application_instance * sectionReader::getRealInstance( const Registry * reg, long int begin, instanceID instance, +SDAI_Application_instance * sectionReader::getRealInstance( const Registry * reg, lazyFileOffset begin, instanceID instance, const std::string & typeName, const std::string & schName, bool header ) { int c; const char * tName = 0, * sName = 0; //these are necessary since typeName and schName are const + std::string normalizedSchema; std::string comment; Severity sev = SEVERITY_NULL; SDAI_Application_instance * inst = 0; @@ -277,7 +297,10 @@ SDAI_Application_instance * sectionReader::getRealInstance( const Registry * reg if( fs ) { StringNode * sn = ( StringNode * ) fs->schema_identifiers_()->GetHead(); if( sn ) { - sName = sn->value.c_str(); + normalizedSchema = sn->value.c_str(); + size_t qualifier = normalizedSchema.find_first_of( " {" ); + if( qualifier != std::string::npos ) normalizedSchema.erase( qualifier ); + sName = normalizedSchema.c_str(); if( sn->NextNode() ) { std::cerr << "Warning - multiple schema names found. Only searching with first one." << std::endl; } @@ -287,7 +310,7 @@ SDAI_Application_instance * sectionReader::getRealInstance( const Registry * reg } } - _file.seekg( begin ); + _file.seekg( static_cast( begin ) ); skipWS(); ReadTokenSeparator( _file, &comment ); if( !header ) { @@ -319,7 +342,7 @@ SDAI_Application_instance * sectionReader::getRealInstance( const Registry * reg inst->AddP21Comment( comment ); } assert( inst->eDesc ); - _file.seekg( begin ); + _file.seekg( static_cast( begin ) ); findNormalString( "(" ); _file.seekg( _file.tellg() - std::streampos(1) ); sev = inst->STEPread( instance, 0, _lazyFile->getInstMgr()->getAdapter(), _file, sName, true, false ); @@ -360,7 +383,6 @@ STEPcomplex * sectionReader::CreateSubSuperInstance( const Registry * reg, insta //TODO still need the schema name STEPcomplex * sc = new STEPcomplex( ( const_cast( reg ) ), names, ( int ) fileid /*, schnm*/ ); delete[] names; - //TODO also delete contents of typeNames! + for( int i = 0; i < s; i++ ) delete typeNames[i]; return sc; } - diff --git a/src/cllazyfile/test/lazy_index.exp b/src/cllazyfile/test/lazy_index.exp new file mode 100644 index 000000000..241a88730 --- /dev/null +++ b/src/cllazyfile/test/lazy_index.exp @@ -0,0 +1,25 @@ +SCHEMA lazy_test_schema; + +ENTITY b; +END_ENTITY; + +ENTITY a; + opt : OPTIONAL STRING; + label : STRING; + ref : b; +END_ENTITY; + +ENTITY c; +END_ENTITY; + +ENTITY d + SUBTYPE OF (c); + ref : a; +END_ENTITY; + +ENTITY e; + ref : d; + opt : OPTIONAL STRING; +END_ENTITY; + +END_SCHEMA; diff --git a/src/cllazyfile/test/lazy_index.stp b/src/cllazyfile/test/lazy_index.stp new file mode 100644 index 000000000..7d3cc8629 --- /dev/null +++ b/src/cllazyfile/test/lazy_index.stp @@ -0,0 +1,17 @@ +ISO-10303-21; +HEADER; +FILE_DESCRIPTION(('lazy index test'),'2;1'); +FILE_NAME('lazy_index.stp','2026-07-17T00:00:00',('STEPcode'),('STEPcode'),'','',''); +FILE_SCHEMA(('LAZY_TEST_SCHEMA { 1 0 10303 999 }')); +ENDSEC; +DATA('second',('LAZY_TEST_SCHEMA')); +#1=A($,'escaped quote '' and unicode \X2\03A9\X0\',#2); +#2=B(); +ENDSEC; +/* A second data section must not be consumed by lookahead. */ +DATA; +#3=(C()D(#1)); +#4=E(#3,$); +#5=E(#99,$); +ENDSEC; +END-ISO-10303-21; From 844a380d4c4686750e88eeab3ab7c24b893e8478 Mon Sep 17 00:00:00 2001 From: Clifford Yapp <238416+starseeker@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:55:48 -0400 Subject: [PATCH 03/11] lazy: recognize Part 21 comments while scanning Skip comments as opaque text during token searches so quoted text and string-control sequences within a comment cannot affect indexing. --- include/cllazyfile/sectionReader.h | 6 +++++ src/cllazyfile/sectionReader.cc | 35 +++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/include/cllazyfile/sectionReader.h b/include/cllazyfile/sectionReader.h index 575fd96d3..f27928efe 100644 --- a/include/cllazyfile/sectionReader.h +++ b/include/cllazyfile/sectionReader.h @@ -46,6 +46,12 @@ class SC_LAZYFILE_EXPORT sectionReader { */ std::streampos findNormalString( const std::string & str, bool semicolon = false ); + /** Skip from immediately after a comment's opening slash and star + * through its closing star and slash. Part 21 comments are opaque: + * apostrophes and string-control text inside them have no syntax. + */ + bool skipComment(); + /** Get a keyword ending with one of delimiters. */ const char * getDelimitedKeyword( const char * delimiters ); diff --git a/src/cllazyfile/sectionReader.cc b/src/cllazyfile/sectionReader.cc index ef4a5c427..e1812603e 100644 --- a/src/cllazyfile/sectionReader.cc +++ b/src/cllazyfile/sectionReader.cc @@ -39,6 +39,19 @@ sectionReader::~sectionReader() { delete _error; } +bool sectionReader::skipComment() { + int previous = 0; + int current; + while( current = _file.get(), _file.good() ) { + if( previous == '*' && current == '/' ) { + return true; + } + previous = current; + } + return false; +} + + std::streampos sectionReader::findNormalString( const std::string & str, bool semicolon ) { std::streampos found = -1, startPos = _file.tellg(), nextTry = startPos; int i = 0, l = str.length(); @@ -63,8 +76,11 @@ std::streampos sectionReader::findNormalString( const std::string & str, bool se GetLiteralStr( _file, _lazyFile->getInstMgr()->getErrorDesc() ); } if( ( c == '/' ) && ( _file.peek() == '*' ) ) { - //push past comment - findNormalString( "*/" ); + _file.get(); // consume the opening star + if( !skipComment() ) { + return -1; + } + continue; } if( str[i] == c ) { i++; @@ -106,7 +122,10 @@ const char * sectionReader::getDelimitedKeyword( const char * delimiters ) { str.append( 1, c ); } else if( ( c == '/' ) && ( _file.peek() == '*' ) && ( str.length() == 0 ) ) { //push past comment - findNormalString( "*/" ); + _file.get(); // consume the opening star + if( !skipComment() ) { + break; + } skipWS(); continue; } else { @@ -137,7 +156,10 @@ std::streampos sectionReader::seekInstanceEnd( instanceRefs ** refs, std::vector break; case '/': if( _file.peek() == '*' ) { - findNormalString( "*/" ); + _file.get(); // consume the opening star + if( !skipComment() ) { + return -1; + } } else { return -1; } @@ -215,7 +237,10 @@ instanceID sectionReader::readInstanceNumber() { skipWS(); c = _file.get(); if( ( c == '/' ) && ( _file.peek() == '*' ) ) { - findNormalString( "*/" ); + _file.get(); // consume the opening star + if( !skipComment() ) { + return 0; + } } else { _file.seekg( _file.tellg() - std::streampos(1) ); } From 66e696471cf1917d7b842b8b22e1559f5046b5c7 Mon Sep 17 00:00:00 2001 From: Clifford Yapp <238416+starseeker@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:56:05 -0400 Subject: [PATCH 04/11] lazy: detect cyclic instance materialization Track in-progress instance loads and report recursive references instead of recursing indefinitely through a cyclic Part 21 graph. --- include/cllazyfile/lazyInstMgr.h | 1 + src/cllazyfile/lazyInstMgr.cc | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/include/cllazyfile/lazyInstMgr.h b/include/cllazyfile/lazyInstMgr.h index 63443a474..20b53e9e4 100644 --- a/include/cllazyfile/lazyInstMgr.h +++ b/include/cllazyfile/lazyInstMgr.h @@ -80,6 +80,7 @@ class SC_LAZYFILE_EXPORT lazyInstMgr { std::map _pinCounts; std::set _batchOwnedInstances; std::set _permanentlyLoadedInstances; + std::set _instancesLoading; size_t _batchLoadDepth; LazyProgressCallback _progressCallback; diff --git a/src/cllazyfile/lazyInstMgr.cc b/src/cllazyfile/lazyInstMgr.cc index 20857ecd1..f19945b3a 100644 --- a/src/cllazyfile/lazyInstMgr.cc +++ b/src/cllazyfile/lazyInstMgr.cc @@ -254,6 +254,33 @@ SDAI_Application_instance * lazyInstMgr::loadInstance( instanceID id, bool reSee emitDiagnostic( diagnostic ); return 0; } + const std::pair::iterator, bool> loading_result = _instancesLoading.insert( id ); + if( !loading_result.second ) { + LazyDiagnostic diagnostic; + diagnostic.severity = LAZY_DIAGNOSTIC_ERROR; + diagnostic.entity = id; + const char * type = typeFromFile( id ); + if( type ) diagnostic.type = type; + instanceStreamPos_t::cvector * positions = _instanceStreamPos.find( id ); + if( positions && !positions->empty() ) { + diagnostic.offset = positions->front().begin; + } + diagnostic.message = "cyclic dependency encountered while materializing instance"; + emitDiagnostic( diagnostic ); + return 0; + } + /* sectionReader materializes referenced instances recursively. Keep an + * explicit in-progress set because an instance is not cacheable until its + * STEPread() has completed. */ + class LoadingGuard { + public: + LoadingGuard( std::set & loading, instanceID id ) + : _loading( loading ), _id( id ) {} + ~LoadingGuard() { _loading.erase( _id ); } + private: + std::set & _loading; + instanceID _id; + } loading_guard( _instancesLoading, id ); instanceStreamPos_t::cvector * cv; if( 0 != ( cv = _instanceStreamPos.find( id ) ) ) { switch( cv->size() ) { From 6d7e383ed206e6e2feeeb06aaaa72ca47115eb38 Mon Sep 17 00:00:00 2001 From: Clifford Yapp <238416+starseeker@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:56:05 -0400 Subject: [PATCH 05/11] lazy: account for resident source record bytes Track indexed record bounds and expose the source-byte footprint of materialized instances in cache statistics. Avoid promoting batch-owned cache hits during reference resolution. --- include/cllazyfile/instMgrHelper.h | 4 +++- include/cllazyfile/lazyInstMgr.h | 5 ++++- include/cllazyfile/lazySupport.h | 5 ++++- include/cllazyfile/lazyTypes.h | 1 + src/cllazyfile/lazyInstMgr.cc | 24 +++++++++++++++++++--- src/cllazyfile/lazyP21DataSectionReader.cc | 4 ++++ src/cllazyfile/p21HeaderSectionReader.cc | 4 ++++ 7 files changed, 41 insertions(+), 6 deletions(-) diff --git a/include/cllazyfile/instMgrHelper.h b/include/cllazyfile/instMgrHelper.h index daa046eb3..881d83ba3 100644 --- a/include/cllazyfile/instMgrHelper.h +++ b/include/cllazyfile/instMgrHelper.h @@ -31,7 +31,9 @@ class SC_LAZYFILE_EXPORT mgrNodeHelper: public MgrNodeBase { _id = id; } inline SDAI_Application_instance * GetSTEPentity() { - return _lim->loadInstance( _id, true ); + /* Attribute resolution must not convert a batch-owned cache hit + * into a process-lifetime retained instance. */ + return _lim->loadInstance( _id, true, false ); } }; diff --git a/include/cllazyfile/lazyInstMgr.h b/include/cllazyfile/lazyInstMgr.h index 20b53e9e4..3a6a7958b 100644 --- a/include/cllazyfile/lazyInstMgr.h +++ b/include/cllazyfile/lazyInstMgr.h @@ -74,10 +74,12 @@ class SC_LAZYFILE_EXPORT lazyInstMgr { unsigned long _lazyInstanceCount, _loadedInstanceCount; uint64_t _cacheHighWater, _cacheHits, _cacheMisses, _materializations, _evictions; uint64_t _activeBatches; + uint64_t _residentSourceBytes, _sourceBytesHighWater; int _longestTypeNameLen; std::string _longestTypeName; std::map _pinCounts; + std::map _instanceSourceBytes; std::set _batchOwnedInstances; std::set _permanentlyLoadedInstances; std::set _instancesLoading; @@ -222,8 +224,9 @@ class SC_LAZYFILE_EXPORT lazyInstMgr { /** returns a pointer to an instance, loading it if necessary. * \param id the instance number to look for * \param reSeek if true, reset file position to current position when done. only necessary when loading an instance with dependencies; excessive use will cause a performance hit + * \param promoteCached if true, a cached instance requested outside a batch is retained for legacy callers */ - SDAI_Application_instance * loadInstance( instanceID id, bool reSeek = false ); + SDAI_Application_instance * loadInstance( instanceID id, bool reSeek = false, bool promoteCached = true ); LazyInstanceBatch loadBatch( instanceID root ); LazyInstanceBatch loadBatch( const std::vector & roots ); diff --git a/include/cllazyfile/lazySupport.h b/include/cllazyfile/lazySupport.h index ce398fea7..79a04018f 100644 --- a/include/cllazyfile/lazySupport.h +++ b/include/cllazyfile/lazySupport.h @@ -54,11 +54,14 @@ struct SC_LAZYFILE_EXPORT LazyCacheStatistics { uint64_t evictions; uint64_t activeBatches; uint64_t dataSections; + uint64_t residentSourceBytes; + uint64_t sourceBytesHighWater; bool cancelled; LazyCacheStatistics(): instancesScanned( 0 ), instancesLoaded( 0 ), instancesPinned( 0 ), cacheHighWater( 0 ), cacheHits( 0 ), cacheMisses( 0 ), materializations( 0 ), - evictions( 0 ), activeBatches( 0 ), dataSections( 0 ), cancelled( false ) {} + evictions( 0 ), activeBatches( 0 ), dataSections( 0 ), residentSourceBytes( 0 ), + sourceBytesHighWater( 0 ), cancelled( false ) {} }; typedef std::function LazyProgressCallback; diff --git a/include/cllazyfile/lazyTypes.h b/include/cllazyfile/lazyTypes.h index df8c7c631..5bcbdcf30 100644 --- a/include/cllazyfile/lazyTypes.h +++ b/include/cllazyfile/lazyTypes.h @@ -61,6 +61,7 @@ typedef std::set< instanceID > instanceSet; */ typedef struct { lazyFileOffset begin; ///< byte offset before the instance ID; whitespace or comments may precede it. + lazyFileOffset end; ///< byte offset immediately after the terminating semicolon. instanceID instance; sectionID section; /* bool modified; */ /* this will be useful when writing instances - if an instance is diff --git a/src/cllazyfile/lazyInstMgr.cc b/src/cllazyfile/lazyInstMgr.cc index f19945b3a..1e6a0a5cc 100644 --- a/src/cllazyfile/lazyInstMgr.cc +++ b/src/cllazyfile/lazyInstMgr.cc @@ -23,6 +23,8 @@ lazyInstMgr::lazyInstMgr() { _materializations = 0; _evictions = 0; _activeBatches = 0; + _residentSourceBytes = 0; + _sourceBytesHighWater = 0; _batchLoadDepth = 0; _cancelled = false; _longestTypeNameLen = 0; @@ -91,6 +93,9 @@ void lazyInstMgr::addLazyInstance( namedLazyInstance inst ) { emitDiagnostic( diagnostic ); } _instanceStreamPos.insert( inst.loc.instance, pos ); + if( !duplicate && inst.loc.end >= inst.loc.begin ) { + _instanceSourceBytes[inst.loc.instance] = inst.loc.end - inst.loc.begin; + } if( inst.refs ) { if( inst.refs->size() > 0 ) { @@ -173,6 +178,8 @@ LazyCacheStatistics lazyInstMgr::cacheStatistics() const { stats.evictions = _evictions; stats.activeBatches = _activeBatches; stats.dataSections = _dataSections.size(); + stats.residentSourceBytes = _residentSourceBytes; + stats.sourceBytesHighWater = _sourceBytesHighWater; stats.cancelled = _cancelled; return stats; } @@ -235,14 +242,14 @@ void lazyInstMgr::validateReferences() { } } -SDAI_Application_instance * lazyInstMgr::loadInstance( instanceID id, bool reSeek ) { +SDAI_Application_instance * lazyInstMgr::loadInstance( instanceID id, bool reSeek, bool promoteCached ) { assert( _mainRegistry && "Main registry has not been initialized. Do so with initRegistry() or setRegistry()." ); std::streampos oldPos; - instancePosition pos; + instancePosition pos = instancePosition(); SDAI_Application_instance * inst = _instancesLoaded.find( id ); if( inst ) { ++_cacheHits; - if( _batchLoadDepth == 0 ) _permanentlyLoadedInstances.insert( id ); + if( promoteCached && _batchLoadDepth == 0 ) _permanentlyLoadedInstances.insert( id ); return inst; } ++_cacheMisses; @@ -314,6 +321,12 @@ SDAI_Application_instance * lazyInstMgr::loadInstance( instanceID id, bool reSee _loadedInstanceCount++; ++_materializations; _cacheHighWater = std::max( _cacheHighWater, _loadedInstanceCount ); + std::map::const_iterator source_size = _instanceSourceBytes.find( id ); + if( source_size != _instanceSourceBytes.end() ) { + _residentSourceBytes += source_size->second; + _sourceBytesHighWater = std::max( _sourceBytesHighWater, + _residentSourceBytes ); + } if( _batchLoadDepth ) { _batchOwnedInstances.insert( id ); } else { @@ -405,6 +418,11 @@ void lazyInstMgr::releaseBatch( const std::vector & instances ) { _batchOwnedInstances.erase( *id ); deleted.push_back( inst ); --_loadedInstanceCount; + std::map::const_iterator source_size = _instanceSourceBytes.find( *id ); + if( source_size != _instanceSourceBytes.end() ) { + _residentSourceBytes = source_size->second <= _residentSourceBytes ? + _residentSourceBytes - source_size->second : 0; + } ++_evictions; } std::vector::iterator inst = deleted.begin(); diff --git a/src/cllazyfile/lazyP21DataSectionReader.cc b/src/cllazyfile/lazyP21DataSectionReader.cc index b768c6138..16b58618f 100644 --- a/src/cllazyfile/lazyP21DataSectionReader.cc +++ b/src/cllazyfile/lazyP21DataSectionReader.cc @@ -98,6 +98,7 @@ const namedLazyInstance lazyP21DataSectionReader::nextInstance() { i.refs = 0; i.componentTypes = 0; i.loc.section = 0; + i.loc.end = 0; std::streampos start = _file.tellg(); i.loc.begin = start == std::streampos( -1 ) ? 0 : static_cast( static_cast( start ) ); @@ -109,6 +110,9 @@ const namedLazyInstance lazyP21DataSectionReader::nextInstance() { if( _file.good() ) { if( i.name[0] == '\0' ) i.componentTypes = new std::vector; end = seekInstanceEnd( & i.refs, i.componentTypes ); + if( end != std::streampos( -1 ) ) { + i.loc.end = static_cast( static_cast( end ) ); + } } } if( ( i.loc.instance == 0 ) || ( !_file.good() ) || ( end == ( std::streampos ) - 1 ) ) { diff --git a/src/cllazyfile/p21HeaderSectionReader.cc b/src/cllazyfile/p21HeaderSectionReader.cc index e11f1b14f..e8fa784d3 100644 --- a/src/cllazyfile/p21HeaderSectionReader.cc +++ b/src/cllazyfile/p21HeaderSectionReader.cc @@ -35,6 +35,7 @@ const namedLazyInstance p21HeaderSectionReader::nextInstance() { i.refs = 0; i.componentTypes = 0; + i.loc.end = 0; std::streampos start = _file.tellg(); i.loc.begin = start == std::streampos( -1 ) ? 0 : static_cast( static_cast( start ) ); @@ -58,6 +59,9 @@ const namedLazyInstance p21HeaderSectionReader::nextInstance() { assert( strlen( i.name ) > 0 ); std::streampos end = seekInstanceEnd( 0 ); //no references in file header + if( end != std::streampos( -1 ) ) { + i.loc.end = static_cast( static_cast( end ) ); + } if( ( (signed long int)end == -1 ) || ( end >= _sectionEnd ) ) { //invalid instance, so clear everything i.loc.begin = 0; From d69363b85ebc56e2425f31692e3692e5ee5da268 Mon Sep 17 00:00:00 2001 From: Clifford Yapp <238416+starseeker@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:56:19 -0400 Subject: [PATCH 06/11] lazy: expose exact indexed source records Retain record extents during indexing and provide stream-safe access to an instance's original Part 21 record without materializing it. --- include/cllazyfile/lazyInstMgr.h | 4 ++++ include/cllazyfile/sectionReader.h | 6 ++++++ src/cllazyfile/lazyInstMgr.cc | 14 ++++++++++++++ src/cllazyfile/lazy_index_test.cc | 10 ++++++++++ src/cllazyfile/sectionReader.cc | 27 +++++++++++++++++++++++++++ 5 files changed, 61 insertions(+) diff --git a/include/cllazyfile/lazyInstMgr.h b/include/cllazyfile/lazyInstMgr.h index 3a6a7958b..c55a13ed6 100644 --- a/include/cllazyfile/lazyInstMgr.h +++ b/include/cllazyfile/lazyInstMgr.h @@ -125,6 +125,10 @@ class SC_LAZYFILE_EXPORT lazyInstMgr { } LazyInstanceIdView forwardReferences( instanceID id ); LazyInstanceIdView reverseReferences( instanceID id ); + /** Copy the exact indexed source record for an instance. Returns an + * empty string when the ID is missing or ambiguous. */ + std::string sourceRecord( instanceID id ); + /// returns a vector containing the instances that match `type` instanceTypes_t::cvector * getInstances( std::string type, bool caseSensitive = false ) { /*const*/ if( !caseSensitive ) { diff --git a/include/cllazyfile/sectionReader.h b/include/cllazyfile/sectionReader.h index f27928efe..28144e67b 100644 --- a/include/cllazyfile/sectionReader.h +++ b/include/cllazyfile/sectionReader.h @@ -72,6 +72,12 @@ class SC_LAZYFILE_EXPORT sectionReader { SDAI_Application_instance * getRealInstance( const Registry * reg, lazyFileOffset begin, instanceID instance, const std::string & typeName = "", const std::string & schName = "", bool header = false ); + /** Return one indexed source record without materializing it. The + * stream position is restored before returning. This is intended + * for lightweight adapters which can detach scalar/reference data + * directly from Part 21 source. */ + std::string sourceRecord( lazyFileOffset begin, uint64_t length ); + sectionID ID() const { return _sectionID; } diff --git a/src/cllazyfile/lazyInstMgr.cc b/src/cllazyfile/lazyInstMgr.cc index 1e6a0a5cc..fa48005b9 100644 --- a/src/cllazyfile/lazyInstMgr.cc +++ b/src/cllazyfile/lazyInstMgr.cc @@ -129,6 +129,20 @@ LazyInstanceIdView lazyInstMgr::reverseReferences( instanceID id ) { return LazyInstanceIdView( _revInstanceRefs.find( id ) ); } +std::string lazyInstMgr::sourceRecord( instanceID id ) { + instanceStreamPos_t::cvector * positions = _instanceStreamPos.find( id ); + std::map::const_iterator bytes = _instanceSourceBytes.find( id ); + if( !positions || positions->size() != 1 || bytes == _instanceSourceBytes.end() || + bytes->second == 0 ) { + return std::string(); + } + const instancePosition & position = positions->front(); + if( position.section >= _dataSections.size() || !_dataSections[position.section] ) { + return std::string(); + } + return _dataSections[position.section]->sourceRecord( position.begin, bytes->second ); +} + void lazyInstMgr::observeScan( fileID file, lazyFileOffset offset, lazyFileOffset fileSize ) { if( _cancelled ) return; if( _cancellationCallback && _cancellationCallback() ) { diff --git a/src/cllazyfile/lazy_index_test.cc b/src/cllazyfile/lazy_index_test.cc index ce59013b8..a7d9c1a9c 100644 --- a/src/cllazyfile/lazy_index_test.cc +++ b/src/cllazyfile/lazy_index_test.cc @@ -46,6 +46,16 @@ int main( int argc, char ** argv ) { "forward reference index failed" ); require( manager.reverseReferences( 1 ).size() == 1 && manager.reverseReferences( 1 )[0] == 3, "reverse reference index failed" ); + const std::string firstSource = manager.sourceRecord( 1 ); + const std::string secondSource = manager.sourceRecord( 2 ); + require( firstSource.find( "#1=A(" ) != std::string::npos && + !firstSource.empty() && firstSource[firstSource.size() - 1] == ';', + "first exact source record failed" ); + require( secondSource.find( "#2=B();" ) != std::string::npos && + !secondSource.empty() && secondSource[secondSource.size() - 1] == ';', + "second exact source record or stream restoration failed" ); + require( manager.sourceRecord( 99 ).empty(), + "missing source record was not empty" ); lazyInstMgr cancelled; uint64_t cancellationCalls = 0; diff --git a/src/cllazyfile/sectionReader.cc b/src/cllazyfile/sectionReader.cc index e1812603e..345d0eb35 100644 --- a/src/cllazyfile/sectionReader.cc +++ b/src/cllazyfile/sectionReader.cc @@ -10,6 +10,7 @@ #include #include #include +#include #ifdef _WIN32 # define strtoull _strtoui64 @@ -108,6 +109,32 @@ std::streampos sectionReader::findNormalString( const std::string & str, bool se } +std::string sectionReader::sourceRecord( lazyFileOffset begin, uint64_t length ) { + if( begin == 0 || length == 0 || + length > static_cast( std::numeric_limits::max() ) || + length > static_cast( std::numeric_limits::max() ) ) { + return std::string(); + } + + const std::streampos saved = _file.tellg(); + _file.clear(); + _file.seekg( static_cast( begin ) ); + if( !_file.good() ) { + _file.clear(); + if( saved != std::streampos( -1 ) ) _file.seekg( saved ); + return std::string(); + } + + std::string source( static_cast( length ), '\0' ); + _file.read( &source[0], static_cast( length ) ); + const bool complete = static_cast( _file.gcount() ) == length; + + _file.clear(); + if( saved != std::streampos( -1 ) ) _file.seekg( saved ); + return complete ? source : std::string(); +} + + //NOTE different behavior than const char * GetKeyword( istream & in, const char * delims, ErrorDescriptor & err ) in read_func.cc // returns pointer to the contents of a static std::string const char * sectionReader::getDelimitedKeyword( const char * delimiters ) { From 9005894a16ed8db1ad91c615222def0f1a027bd7 Mon Sep 17 00:00:00 2001 From: Clifford Yapp <238416+starseeker@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:56:19 -0400 Subject: [PATCH 07/11] lazy: allow explicit materialization type aliases Permit callers to map known, attribute-compatible source keywords during object creation while preserving the original keyword in the lazy index. --- include/cllazyfile/lazyInstMgr.h | 23 +++++++++++++++++++++++ src/cllazyfile/sectionReader.cc | 5 +++++ 2 files changed, 28 insertions(+) diff --git a/include/cllazyfile/lazyInstMgr.h b/include/cllazyfile/lazyInstMgr.h index c55a13ed6..bc62322ec 100644 --- a/include/cllazyfile/lazyInstMgr.h +++ b/include/cllazyfile/lazyInstMgr.h @@ -1,6 +1,7 @@ #ifndef LAZYINSTMGR_H #define LAZYINSTMGR_H +#include #include #include #include @@ -80,6 +81,7 @@ class SC_LAZYFILE_EXPORT lazyInstMgr { std::map _pinCounts; std::map _instanceSourceBytes; + std::map _materializationTypeAliases; std::set _batchOwnedInstances; std::set _permanentlyLoadedInstances; std::set _instancesLoading; @@ -112,6 +114,27 @@ class SC_LAZYFILE_EXPORT lazyInstMgr { return ( InstMgrBase * ) _ima; } + /** Register an explicit source-keyword substitution used only when + * materializing an SDAI object. The lazy type index retains the + * original Part 21 keyword. This is intended for known, + * attribute-compatible exporter aliases, not arbitrary recovery. */ + void setMaterializationTypeAlias( std::string source, const std::string & target ) { + for( std::string::iterator c = source.begin(); c != source.end(); ++c ) { + *c = static_cast( toupper( static_cast( *c ) ) ); + } + _materializationTypeAliases[source] = target; + } + + std::string materializationType( std::string source ) const { + std::string key = source; + for( std::string::iterator c = key.begin(); c != key.end(); ++c ) { + *c = static_cast( toupper( static_cast( *c ) ) ); + } + std::map::const_iterator alias = + _materializationTypeAliases.find( key ); + return alias == _materializationTypeAliases.end() ? source : alias->second; + } + instanceRefs_t * getFwdRefs() { return & _fwdInstanceRefs; } diff --git a/src/cllazyfile/sectionReader.cc b/src/cllazyfile/sectionReader.cc index 345d0eb35..407c96b15 100644 --- a/src/cllazyfile/sectionReader.cc +++ b/src/cllazyfile/sectionReader.cc @@ -386,6 +386,11 @@ SDAI_Application_instance * sectionReader::getRealInstance( const Registry * reg if( ( !header ) && ( typeName.size() == 0 ) ) { tName = getDelimitedKeyword( ";( /\\" ); } + std::string materializationType; + if( !header && tName ) { + materializationType = _lazyFile->getInstMgr()->materializationType( tName ); + tName = materializationType.c_str(); + } inst = reg->ObjCreate( tName, sName ); break; } From a267616a30c91512ce5dd697a7587b97485c129c Mon Sep 17 00:00:00 2001 From: Clifford Yapp <238416+starseeker@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:56:19 -0400 Subject: [PATCH 08/11] lazy: stop batch materialization promptly on cancellation Check cancellation while finding dependencies, pinning records, and resolving recursive references. Discard an object if cancellation interrupts its materialization. --- src/cllazyfile/lazyInstMgr.cc | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/cllazyfile/lazyInstMgr.cc b/src/cllazyfile/lazyInstMgr.cc index fa48005b9..651dd28d5 100644 --- a/src/cllazyfile/lazyInstMgr.cc +++ b/src/cllazyfile/lazyInstMgr.cc @@ -257,6 +257,11 @@ void lazyInstMgr::validateReferences() { } SDAI_Application_instance * lazyInstMgr::loadInstance( instanceID id, bool reSeek, bool promoteCached ) { + if( _batchLoadDepth && ( _cancelled || + ( _cancellationCallback && _cancellationCallback() ) ) ) { + _cancelled = true; + return 0; + } assert( _mainRegistry && "Main registry has not been initialized. Do so with initRegistry() or setRegistry()." ); std::streampos oldPos; instancePosition pos = instancePosition(); @@ -318,6 +323,15 @@ SDAI_Application_instance * lazyInstMgr::loadInstance( instanceID id, bool reSee if( reSeek ) { _dataSections[pos.section]->seekg( oldPos ); } + /* A recursive reference load may observe cancellation after + * this instance began materializing. Never cache that + * partially resolved SDAI object. */ + if( _batchLoadDepth && ( _cancelled || + ( _cancellationCallback && _cancellationCallback() ) ) ) { + _cancelled = true; + if( !isNilSTEPentity( inst ) ) delete inst; + inst = 0; + } break; default: std::cerr << "Instance #" << id << " exists in multiple sections. This is not yet supported." << std::endl; @@ -377,6 +391,10 @@ std::vector lazyInstMgr::dependencyClosure( const std::vector queue( roots.begin(), roots.end() ); size_t current = 0; while( current < queue.size() ) { + if( _cancelled || ( _cancellationCallback && _cancellationCallback() ) ) { + _cancelled = true; + break; + } instanceRefs_t::cvector * refs = _fwdInstanceRefs.find( queue[current++] ); if( !refs ) continue; instanceRefs_t::cvector::const_iterator ref = refs->begin(); @@ -395,10 +413,21 @@ LazyInstanceBatch lazyInstMgr::loadBatch( instanceID root ) { LazyInstanceBatch lazyInstMgr::loadBatch( const std::vector & roots ) { std::vector closure = dependencyClosure( roots ); std::vector::const_iterator id = closure.begin(); - for( ; id != closure.end(); ++id ) ++_pinCounts[*id]; + for( ; id != closure.end(); ++id ) { + if( _cancelled || ( _cancellationCallback && _cancellationCallback() ) ) { + _cancelled = true; + closure.erase( id, closure.end() ); + break; + } + ++_pinCounts[*id]; + } ++_activeBatches; ++_batchLoadDepth; for( id = closure.begin(); id != closure.end(); ++id ) { + if( _cancelled || ( _cancellationCallback && _cancellationCallback() ) ) { + _cancelled = true; + break; + } if( !loadInstance( *id, true ) ) { LazyDiagnostic diagnostic; diagnostic.severity = LAZY_DIAGNOSTIC_ERROR; From 0e091f0de7b9aa2fb9fa9be64b49f240327dfeb8 Mon Sep 17 00:00:00 2001 From: Clifford Yapp <238416+starseeker@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:56:47 -0400 Subject: [PATCH 09/11] lazy: index Edition 1 SCOPE records Handle nested SCOPE constructs and export lists while indexing Part 21 records, preserving scope-owned source records and their references. Add scoped-record coverage. --- include/cllazyfile/lazyP21DataSectionReader.h | 3 + include/cllazyfile/sectionReader.h | 17 ++- src/cllazyfile/CMakeLists.txt | 4 +- src/cllazyfile/lazyP21DataSectionReader.cc | 31 ++++- src/cllazyfile/lazy_index_test.cc | 31 ++++- src/cllazyfile/sectionReader.cc | 112 +++++++++++++++++- src/cllazyfile/test/lazy_scope.stp | 17 +++ 7 files changed, 206 insertions(+), 9 deletions(-) create mode 100644 src/cllazyfile/test/lazy_scope.stp diff --git a/include/cllazyfile/lazyP21DataSectionReader.h b/include/cllazyfile/lazyP21DataSectionReader.h index 9086630ec..f24c2802e 100644 --- a/include/cllazyfile/lazyP21DataSectionReader.h +++ b/include/cllazyfile/lazyP21DataSectionReader.h @@ -7,6 +7,9 @@ class SC_LAZYFILE_EXPORT lazyP21DataSectionReader: public lazyDataSectionReader { protected: + /** Index the instances in an Edition 1 SCOPE and leave the stream at + * the owning entity's record. */ + bool indexScope(); public: lazyP21DataSectionReader( lazyFileReader * parent, std::ifstream & file, std::streampos start, sectionID sid ); diff --git a/include/cllazyfile/sectionReader.h b/include/cllazyfile/sectionReader.h index 28144e67b..4665bb765 100644 --- a/include/cllazyfile/sectionReader.h +++ b/include/cllazyfile/sectionReader.h @@ -52,6 +52,19 @@ class SC_LAZYFILE_EXPORT sectionReader { */ bool skipComment(); + /** Skip whitespace and Part 21 comments, leaving the next syntactic + * token unread. */ + bool skipTokenSeparators(); + + /** Skip an Edition 1 SCOPE construct, including nested scopes and its + * optional export list. The stream must initially point at '&' and + * is left at the owning entity's record. */ + bool skipScope(); + + /** Skip an optional scope export list at the current stream + * position. */ + bool skipScopeExportList(); + /** Get a keyword ending with one of delimiters. */ const char * getDelimitedKeyword( const char * delimiters ); @@ -109,7 +122,9 @@ class SC_LAZYFILE_EXPORT sectionReader { } _file.seekg( static_cast( offset ) ); readInstanceNumber(); - skipWS(); + if( !skipTokenSeparators() ) return 0; + if( _file.peek() == '&' && !skipScope() ) return 0; + if( !skipTokenSeparators() ) return 0; return getDelimitedKeyword( ";( /\\" ); } diff --git a/src/cllazyfile/CMakeLists.txt b/src/cllazyfile/CMakeLists.txt index 89b84ee06..b45c40462 100644 --- a/src/cllazyfile/CMakeLists.txt +++ b/src/cllazyfile/CMakeLists.txt @@ -53,5 +53,7 @@ if(SC_ENABLE_TESTING) SC_ADDEXEC(lazy_index_test SOURCES lazy_index_test.cc LINK_LIBRARIES ${_lazy_index_libs} NO_INSTALL) add_test(NAME lazy_index - COMMAND lazy_index_test ${CMAKE_CURRENT_SOURCE_DIR}/test/lazy_index.stp) + COMMAND lazy_index_test + ${CMAKE_CURRENT_SOURCE_DIR}/test/lazy_index.stp + ${CMAKE_CURRENT_SOURCE_DIR}/test/lazy_scope.stp) endif() diff --git a/src/cllazyfile/lazyP21DataSectionReader.cc b/src/cllazyfile/lazyP21DataSectionReader.cc index 16b58618f..a2c201923 100644 --- a/src/cllazyfile/lazyP21DataSectionReader.cc +++ b/src/cllazyfile/lazyP21DataSectionReader.cc @@ -104,9 +104,17 @@ const namedLazyInstance lazyP21DataSectionReader::nextInstance() { static_cast( static_cast( start ) ); i.loc.instance = readInstanceNumber(); if( ( _file.good() ) && ( i.loc.instance > 0 ) ) { - skipWS(); + if( !skipTokenSeparators() ) { + _file.setstate( std::ios::failbit ); + } + if( _file.good() && _file.peek() == '&' && !indexScope() ) { + _file.setstate( std::ios::failbit ); + } + if( _file.good() && !skipTokenSeparators() ) { + _file.setstate( std::ios::failbit ); + } i.loc.section = _sectionID; - i.name = getDelimitedKeyword( ";( /\\" ); + if( _file.good() ) i.name = getDelimitedKeyword( ";( /\\" ); if( _file.good() ) { if( i.name[0] == '\0' ) i.componentTypes = new std::vector; end = seekInstanceEnd( & i.refs, i.componentTypes ); @@ -130,3 +138,22 @@ const namedLazyInstance lazyP21DataSectionReader::nextInstance() { } return i; } +bool lazyP21DataSectionReader::indexScope() { + if( !skipTokenSeparators() || _file.get() != '&' ) return false; + const std::string scopeKeyword = getDelimitedKeyword( "#/(;\\" ); + if( scopeKeyword != "SCOPE" ) return false; + + bool haveInstance = false; + while( skipTokenSeparators() && _file.peek() == '#' ) { + namedLazyInstance nested = nextInstance(); + if( nested.loc.begin == 0 || !nested.name ) return false; + _lazyFile->getInstMgr()->addLazyInstance( nested ); + haveInstance = true; + } + if( !haveInstance || !skipTokenSeparators() ) return false; + + const std::string endKeyword = getDelimitedKeyword( "/#(;\\" ); + if( endKeyword != "ENDSCOPE" ) return false; + return skipScopeExportList(); +} + diff --git a/src/cllazyfile/lazy_index_test.cc b/src/cllazyfile/lazy_index_test.cc index a7d9c1a9c..cb804963e 100644 --- a/src/cllazyfile/lazy_index_test.cc +++ b/src/cllazyfile/lazy_index_test.cc @@ -13,7 +13,7 @@ void require( bool condition, const char * message ) { } int main( int argc, char ** argv ) { - require( argc == 2, "expected fixture path" ); + require( argc == 3, "expected ordinary and scoped fixture paths" ); lazyInstMgr manager; uint64_t progressCalls = 0; @@ -66,5 +66,34 @@ int main( int argc, char ** argv ) { stats = cancelled.cacheStatistics(); require( stats.cancelled, "cancellation was not recorded" ); require( stats.instancesScanned == 2, "scan did not stop at cancellation boundary" ); + + lazyInstMgr scoped; + require( scoped.openFile( argv[2] ), "scoped fixture did not open" ); + stats = scoped.cacheStatistics(); + require( stats.instancesScanned == 5, "wrong scoped scan count" ); + require( scoped.instancesByType( "A" ).size() == 3, + "scoped simple instances were not indexed" ); + require( scoped.instancesByType( "B" ).size() == 1, + "nested scoped instance was not indexed" ); + require( scoped.instancesByType( "C" ).size() == 1 && + scoped.instancesByType( "D" ).size() == 1 && + scoped.instancesByType( "" ).size() == 1, + "scoped complex owner was not indexed" ); + require( scoped.forwardReferences( 11 ).size() == 1 && + scoped.forwardReferences( 11 )[0] == 12, + "nested scoped reference was not indexed" ); + require( scoped.forwardReferences( 20 ).size() == 1 && + scoped.forwardReferences( 20 )[0] == 21, + "complex scope-owner reference was not indexed" ); + const std::string nestedOwner = scoped.sourceRecord( 11 ); + const std::string complexOwner = scoped.sourceRecord( 20 ); + require( nestedOwner.find( "#11=&SCOPE" ) != std::string::npos && + nestedOwner.find( "ENDSCOPE /#12/ A(" ) != std::string::npos && + !nestedOwner.empty() && nestedOwner[nestedOwner.size() - 1] == ';', + "nested scope source record was not preserved" ); + require( complexOwner.find( "#20=&SCOPE" ) != std::string::npos && + complexOwner.find( "ENDSCOPE (C()D(" ) != std::string::npos && + !complexOwner.empty() && complexOwner[complexOwner.size() - 1] == ';', + "complex scope source record was not preserved" ); return EXIT_SUCCESS; } diff --git a/src/cllazyfile/sectionReader.cc b/src/cllazyfile/sectionReader.cc index 407c96b15..e22014598 100644 --- a/src/cllazyfile/sectionReader.cc +++ b/src/cllazyfile/sectionReader.cc @@ -53,6 +53,99 @@ bool sectionReader::skipComment() { } +bool sectionReader::skipTokenSeparators() { + while( _file.good() ) { + skipWS(); + if( _file.peek() != '/' ) return _file.good(); + const std::streampos slash = _file.tellg(); + _file.get(); + if( _file.peek() != '*' ) { + _file.seekg( slash ); + return _file.good(); + } + _file.get(); // consume the opening star + if( !skipComment() ) return false; + } + return false; +} + + +bool sectionReader::skipScopeExportList() { + if( !skipTokenSeparators() ) return false; + if( _file.peek() != '/' ) return true; + _file.get(); + + bool haveExport = false; + while( _file.good() ) { + if( !skipTokenSeparators() || _file.get() != '#' ) return false; + bool haveDigit = false; + bool haveNonzeroDigit = false; + while( _file.good() && isdigit( _file.peek() ) ) { + const int digit = _file.get(); + haveDigit = true; + if( digit != '0' ) haveNonzeroDigit = true; + } + if( !haveDigit || !haveNonzeroDigit || !skipTokenSeparators() ) return false; + haveExport = true; + const int delimiter = _file.get(); + if( delimiter == '/' ) return haveExport; + if( delimiter != ',' ) return false; + } + return false; +} + + +bool sectionReader::skipScope() { + if( !skipTokenSeparators() || _file.get() != '&' ) return false; + skipWS(); + static const char scopeKeyword[] = "SCOPE"; + for( size_t i = 0; scopeKeyword[i]; ++i ) { + if( _file.get() != scopeKeyword[i] ) return false; + } + + int depth = 1; + while( depth > 0 && _file.good() ) { + const int current = _file.get(); + if( current == '\'' ) { + _file.seekg( _file.tellg() - std::streampos( 1 ) ); + GetLiteralStr( _file, _lazyFile->getInstMgr()->getErrorDesc() ); + continue; + } + if( current == '/' && _file.peek() == '*' ) { + _file.get(); + if( !skipComment() ) return false; + continue; + } + if( current == '&' ) { + const std::streampos afterAmpersand = _file.tellg(); + skipWS(); + std::string keyword; + while( _file.good() && ( isupper( _file.peek() ) || + isdigit( _file.peek() ) || _file.peek() == '_' || + _file.peek() == '-' ) ) { + keyword.push_back( static_cast( _file.get() ) ); + } + if( keyword == "SCOPE" ) { + ++depth; + } else { + _file.seekg( afterAmpersand ); + } + continue; + } + if( isupper( current ) ) { + std::string keyword( 1, static_cast( current ) ); + while( _file.good() && ( isupper( _file.peek() ) || + isdigit( _file.peek() ) || _file.peek() == '_' || + _file.peek() == '-' ) ) { + keyword.push_back( static_cast( _file.get() ) ); + } + if( keyword == "ENDSCOPE" ) --depth; + } + } + return depth == 0 && skipScopeExportList(); +} + + std::streampos sectionReader::findNormalString( const std::string & str, bool semicolon ) { std::streampos found = -1, startPos = _file.tellg(), nextTry = startPos; int i = 0, l = str.length(); @@ -371,11 +464,11 @@ SDAI_Application_instance * sectionReader::getRealInstance( const Registry * reg skipWS(); ReadTokenSeparator( _file, &comment ); c = _file.peek(); + if( c == '&' ) { + if( !skipScope() || !skipTokenSeparators() ) return 0; + c = _file.peek(); + } switch( c ) { - case '&': - std::cerr << "Can't handle scope instances. Skipping #" << instance << ", offset " << _file.tellg() << std::endl; - // sev = CreateScopeInstances( in, &scopelist ); - break; case '(': inst = CreateSubSuperInstance( reg, instance, sev ); break; @@ -400,6 +493,17 @@ SDAI_Application_instance * sectionReader::getRealInstance( const Registry * reg } assert( inst->eDesc ); _file.seekg( static_cast( begin ) ); + if( !header ) { + if( findNormalString( "=" ) == std::streampos( -1 ) || + !skipTokenSeparators() ) { + delete inst; + return 0; + } + if( _file.peek() == '&' && ( !skipScope() || !skipTokenSeparators() ) ) { + delete inst; + return 0; + } + } findNormalString( "(" ); _file.seekg( _file.tellg() - std::streampos(1) ); sev = inst->STEPread( instance, 0, _lazyFile->getInstMgr()->getAdapter(), _file, sName, true, false ); diff --git a/src/cllazyfile/test/lazy_scope.stp b/src/cllazyfile/test/lazy_scope.stp new file mode 100644 index 000000000..fdff89c40 --- /dev/null +++ b/src/cllazyfile/test/lazy_scope.stp @@ -0,0 +1,17 @@ +ISO-10303-21; +HEADER; +FILE_DESCRIPTION(('lazy scope test'),'1;1'); +FILE_NAME('lazy_scope.stp','2026-07-31T00:00:00',('STEPcode'),('STEPcode'),'','',''); +FILE_SCHEMA(('LAZY_TEST_SCHEMA { 1 0 10303 999 }')); +ENDSEC; +DATA; +#10=&SCOPE +#11=&SCOPE +#12=B(); +ENDSCOPE /#12/ A($,'nested owner',#12); +ENDSCOPE /#11/ A($,'outer owner',#12); +#20=&SCOPE +#21=A($,'complex child',#12); +ENDSCOPE (C()D(#21)); +ENDSEC; +END-ISO-10303-21; From e7b231ca0718e522777acde0c099a2019281690d Mon Sep 17 00:00:00 2001 From: Clifford Yapp <238416+starseeker@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:05:58 -0400 Subject: [PATCH 10/11] Correct AP209/AP214 crashes The 35 AP209/AP214 crashes came from inverse resolution inspecting an instance still being materialized. The loader now defers such inverse passes until explicit attributes are complete, avoids partial duplicate inverse population, and guards invalid/null referrers. See src/cllazyfile/lazyInstMgr.cc:301 and src/cllazyfile/lazyRefs.h:90. --- src/cllazyfile/lazyInstMgr.cc | 32 ++++++++++++++++++++++++++++++-- src/cllazyfile/lazyRefs.h | 31 ++++++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/src/cllazyfile/lazyInstMgr.cc b/src/cllazyfile/lazyInstMgr.cc index 651dd28d5..ad42458f7 100644 --- a/src/cllazyfile/lazyInstMgr.cc +++ b/src/cllazyfile/lazyInstMgr.cc @@ -301,11 +301,18 @@ SDAI_Application_instance * lazyInstMgr::loadInstance( instanceID id, bool reSee class LoadingGuard { public: LoadingGuard( std::set & loading, instanceID id ) - : _loading( loading ), _id( id ) {} - ~LoadingGuard() { _loading.erase( _id ); } + : _loading( loading ), _id( id ), _active( true ) {} + ~LoadingGuard() { release(); } + void release() { + if( _active ) { + _loading.erase( _id ); + _active = false; + } + } private: std::set & _loading; instanceID _id; + bool _active; } loading_guard( _instancesLoading, id ); instanceStreamPos_t::cvector * cv; if( 0 != ( cv = _instanceStreamPos.find( id ) ) ) { @@ -344,6 +351,10 @@ SDAI_Application_instance * lazyInstMgr::loadInstance( instanceID id, bool reSee } break; } + /* Explicit attributes are now complete. Inverse discovery may + * safely revisit this instance without tripping the materialization + * cycle detector. */ + loading_guard.release(); if( !isNilSTEPentity( inst ) ) { _instancesLoaded.insert( id, inst ); _loadedInstanceCount++; @@ -361,6 +372,7 @@ SDAI_Application_instance * lazyInstMgr::loadInstance( instanceID id, bool reSee _permanentlyLoadedInstances.insert( id ); lazyRefs lr( this, inst ); lazyRefs::referentInstances_t insts = lr.result(); + resolveDeferredInverses(); } } else { std::cerr << "Error loading instance #" << id << "." << std::endl; @@ -382,6 +394,22 @@ SDAI_Application_instance * lazyInstMgr::loadInstance( instanceID id, bool reSee return inst; } +void lazyInstMgr::resolveDeferredInverses() { + if( !_instancesLoading.empty() ) return; + + while( !_deferredInverseInstances.empty() ) { + std::set deferred; + deferred.swap( _deferredInverseInstances ); + std::set::const_iterator id = deferred.begin(); + for( ; id != deferred.end(); ++id ) { + SDAI_Application_instance * inst = cachedInstance( *id ); + if( inst ) { + lazyRefs refs( this, inst ); + } + } + } +} + SDAI_Application_instance * lazyInstMgr::cachedInstance( instanceID id ) { return _instancesLoaded.find( id ); } diff --git a/src/cllazyfile/lazyRefs.h b/src/cllazyfile/lazyRefs.h index b8daff295..d005bc01b 100644 --- a/src/cllazyfile/lazyRefs.h +++ b/src/cllazyfile/lazyRefs.h @@ -87,6 +87,16 @@ class SC_LAZYFILE_EXPORT lazyRefs { } //3c - for each item in both _refMap and edL, add it to _referentInstances potentialReferentInsts( edL ); + /* Do not partially populate an inverse attribute and then repeat + * the whole pass. A referrer that is still being parsed cannot + * be inspected reliably, so defer this instance as a unit. */ + referentInstances_t::const_iterator pending = _referentInstances.begin(); + for( ; pending != _referentInstances.end(); ++pending ) { + if( _lim->isMaterializing( *pending ) ) { + _lim->deferInverseResolution( _id ); + return; + } + } //3d - load each inst iAstruct ias = invAttr( _inst, ia ); referentInstances_t::iterator insts = _referentInstances.begin(); @@ -98,7 +108,12 @@ class SC_LAZYFILE_EXPORT lazyRefs { void loadInstIFFreferent( instanceID inst, iAstruct ias, const Inverse_attribute * ia ) { bool prevLoaded = _lim->isLoaded( inst ); + if( _lim->isMaterializing( inst ) ) { + _lim->deferInverseResolution( _id ); + return; + } SDAI_Application_instance * rinst = _lim->loadInstance( inst ); + if( !rinst ) return; bool ref = refersToCurrentInst( ia, rinst ); if( ref ) { if( ia->inverted_attr_()->IsAggrType() ) { @@ -108,8 +123,16 @@ class SC_LAZYFILE_EXPORT lazyRefs { assert( invAttr( _inst, ia ).a == ias.a ); } EntityAggregate * ea = ias.a; - //TODO check if duplicate - ea->AddNode( new EntityNode( rinst ) ); + bool duplicate = false; + EntityNode * existing = static_cast( ea->GetHead() ); + while( existing ) { + if( existing->node == rinst ) { + duplicate = true; + break; + } + existing = static_cast( existing->NextNode() ); + } + if( !duplicate ) ea->AddNode( new EntityNode( rinst ) ); } else { SDAI_Application_instance * ai = ias.i; if( !ai ) { @@ -131,8 +154,10 @@ class SC_LAZYFILE_EXPORT lazyRefs { ///3e - check if actually inverse ref bool refersToCurrentInst( const Inverse_attribute * ia, SDAI_Application_instance * referrer ) { + if( !referrer ) return false; //find the attr int rindex = attrIndex( referrer, ia->_inverted_attr_id, ia->_inverted_entity_id ); + if( rindex < 0 ) return false; STEPattribute sa = referrer->attributes[ rindex ]; assert( sa.getADesc()->BaseType() == ENTITY_TYPE ); bool found = false; @@ -279,7 +304,7 @@ class SC_LAZYFILE_EXPORT lazyRefs { // 1. find inverse attrs with recursion - getInverseAttrs( ai->eDesc, _iaList ); + getInverseAttrs( _inst->eDesc, _iaList ); //2. find reverse refs, map id to type (stop if there are no inverse attrs or no refs) if( _iaList.size() == 0 || !mapRefsToTypes() ) { From 84fa417d0d637c880660773cea3f7a24ba596078 Mon Sep 17 00:00:00 2001 From: Clifford Yapp <238416+starseeker@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:35:10 -0400 Subject: [PATCH 11/11] Whoopes, missed header changes --- include/cllazyfile/lazyInstMgr.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/cllazyfile/lazyInstMgr.h b/include/cllazyfile/lazyInstMgr.h index bc62322ec..aac042800 100644 --- a/include/cllazyfile/lazyInstMgr.h +++ b/include/cllazyfile/lazyInstMgr.h @@ -23,6 +23,7 @@ class Registry; class instMgrAdapter; +class lazyRefs; class SC_LAZYFILE_EXPORT lazyInstMgr { protected: @@ -85,6 +86,7 @@ class SC_LAZYFILE_EXPORT lazyInstMgr { std::set _batchOwnedInstances; std::set _permanentlyLoadedInstances; std::set _instancesLoading; + std::set _deferredInverseInstances; size_t _batchLoadDepth; LazyProgressCallback _progressCallback; @@ -96,7 +98,15 @@ class SC_LAZYFILE_EXPORT lazyInstMgr { instMgrAdapter * _ima; friend class LazyInstanceBatch; + friend class lazyRefs; void releaseBatch( const std::vector & instances ); + void resolveDeferredInverses(); + bool isMaterializing( instanceID id ) const { + return _instancesLoading.count( id ) != 0; + } + void deferInverseResolution( instanceID id ) { + _deferredInverseInstances.insert( id ); + } SDAI_Application_instance * cachedInstance( instanceID id ); std::vector dependencyClosure( const std::vector & roots );