/// Creates a new catalog from parquet files.
///
/// Users are required to [wipe](crate::catalog::PreservedCatalog::wipe) the existing catalog before running this
/// procedure (**after creating a backup!**).
///
/// # Limitations
/// Compared to an intact catalog, wiping a catalog and rebuilding it from Parquet files has the following drawbacks:
///
/// - **Garbage Susceptibility:** The rebuild process will stumble over garbage parquet files (i.e. files being present
///   in the object store but that were not part of the catalog). For files that where not written by IOx it will likely
///   report [`Error::MetadataReadFailure`]. For files that are left-overs from previous transactions it will likely
///   report [`Error::MultipleTransactionsFailure`]. Crafted files (i.e. files with the correct metadata and matching
///   transaction UUIDs) will blindly be included into the new catalog, because we have no way to distinguish them from
///   the actual catalog content.
/// - **No Removals:** The rebuild system cannot recover the fact that files where removed from the catalog during some
///   transaction. This might not always be an issue due to "deduplicate while read"-logic in the query engine, but also
///   might have unwanted side effects (e.g. performance issues).
///
/// # Error Handling
/// This routine will fail if:
///
/// - **Metadata Read Failure:** There is a parquet file with metadata that cannot be read. Set
///   `ignore_metadata_read_failure` to `true` to ignore these cases.
/// - **Parquet With Revision Zero:** One of the parquet files reports it belongs to revision `0`. This should never
///   happen since the first transaction is always an empty one. This was likely causes by a bug or a file created by
///   3rd party tooling.
/// - **Multiple Transactions:** If there are multiple transaction with the same revision but different UUIDs, this
///   routine cannot reconstruct a single linear revision history. Make sure to
//    [clean up](crate::cleanup::cleanup_unreferenced_parquet_files) regularly to avoid this case.
Creates a new catalog from parquet files.
Creates a new catalog from parquet files .
pub async fn rebuild_catalog<S, N>(
    object_store: Arc<ObjectStore>,
    search_location: &Path,
    server_id: ServerId,
    db_name: N,
    catalog_empty_input: S::EmptyInput,
    ignore_metadata_read_failure: bool,
) -> Result<PreservedCatalog<S>>
where
    S: CatalogState,
    N: Into<String>,
{
    // collect all revisions from parquet files
    let revisions =
        collect_revisions(&object_store, search_location, ignore_metadata_read_failure).await?;
    // create new empty catalog
    let catalog =
        PreservedCatalog::<S>::new_empty(object_store, server_id, db_name, catalog_empty_input)
            .await
            .context(NewEmptyFailure)?;
    // simulate all transactions
    if let Some(max_revision) = revisions.keys().max() {
        for revision_counter in 1..=*max_revision {
            assert_eq!(
                catalog.revision_counter() + 1,
                revision_counter,
                "revision counter during transaction simulation out-of-sync"
            );
            if let Some((uuid, entries)) = revisions.get(&revision_counter) {
                // we have files for this particular transaction
                let mut transaction = catalog.open_transaction_with_uuid(*uuid).await;
                for (path, metadata) in entries {
                    let path: DirsAndFileName = path.clone().into();
                    transaction
                        .add_parquet(&path, metadata)
                        .context(FileRecordFailure)?;
                }
                transaction.commit().await.context(CommitFailure)?;
            } else {
                // we do not have any files for this transaction (there might have been other actions though or it was
                // an empty transaction) => create new empty transaction
                let transaction = catalog.open_transaction().await;
                transaction.commit().await.context(CommitFailure)?;
            }
        }
    }
    Ok(catalog)
}
pub async fn rebuild_catalog < S , N > ( object_store : Arc < ObjectStore > , search_location : & Path , server_id : ServerId , db_name : N , catalog_empty_input : S :: EmptyInput , ignore_metadata_read_failure : bool , ) -> Result < PreservedCatalog < S > > where S : CatalogState , N : Into < String > , { // collect all revisions from parquet files let revisions = collect_revisions ( & object_store , search_location , ignore_metadata_read_failure ) . await ? ; // create new empty catalog let catalog = PreservedCatalog :: < S > :: new_empty ( object_store , server_id , db_name , catalog_empty_input ) . await . context ( NewEmptyFailure ) ? ; // simulate all transactions if let Some ( max_revision ) = revisions . keys ( ) . max ( ) { for revision_counter in 1 ..= * max_revision { assert_eq ! ( catalog revision_counter ( ) 1 revision_counter " " ) ; if let Some ( ( uuid , entries ) ) = revisions . get ( & revision_counter ) { // we have files for this particular transaction let mut transaction = catalog . open_transaction_with_uuid ( * uuid ) . await ; for ( path , metadata ) in entries { let path : DirsAndFileName = path . clone ( ) . into ( ) ; transaction . add_parquet ( & path , metadata ) . context ( FileRecordFailure ) ? ; } transaction . commit ( ) . await . context ( CommitFailure ) ? ; } else { // we do not have any files for this transaction (there might have been other actions though or it was // an empty transaction) => create new empty transaction let transaction = catalog . open_transaction ( ) . await ; transaction . commit ( ) . await . context ( CommitFailure ) ? ; } } } Ok ( catalog ) }
Creates a new catalog from parquet files.
Creates a new catalog from parquet files .
object_store
Arc<ObjectStore>
search_location
&Path
server_id
ServerId
db_name
N
catalog_empty_input
S::EmptyInput
ignore_metadata_read_failure
bool
limitations
Compared to an intact catalog, wiping a catalog and rebuilding it from Parquet files has the following drawbacks.

Garbage Susceptibility:** The rebuild process will stumble over garbage parquet files . For files that where not written by IOx it will likely
report [`Error::MetadataReadFailure`]. For files that are left-overs from previous transactions it will likely
report [`Error::MultipleTransactionsFailure`]. Crafted files  will blindly be included into the new catalog, because we have no way to distinguish them from
the actual catalog content.
No Removals:** The rebuild system cannot recover the fact that files where removed from the catalog during some
transaction. This might not always be an issue due to "deduplicate while read"-logic in the query engine, but also
might have unwanted side effects .
Compared to an intact catalog wiping a catalog and rebuilding it from Parquet files has the following drawbacks . Garbage Susceptibility : ** The rebuild process will stumble over garbage parquet files . For files that where not written by IOx it will likely report [ ` Error :: MetadataReadFailure ` ] . For files that are left - overs from previous transactions it will likely report [ ` Error :: MultipleTransactionsFailure ` ] . Crafted files will blindly be included into the new catalog because we have no way to distinguish them from the actual catalog content . No Removals : ** The rebuild system cannot recover the fact that files where removed from the catalog during some transaction . This might not always be an issue due to " deduplicate while read " - logic in the query engine but also might have unwanted side effects .
error handling
This routine will fail if.

Metadata Read Failure:** There is a parquet file with metadata that cannot be read. Set
`ignore_metadata_read_failure` to `true` to ignore these cases.
Parquet With Revision Zero:** One of the parquet files reports it belongs to revision `0`. This should never
happen since the first transaction is always an empty one. This was likely causes by a bug or a file created by
3rd party tooling.
Multiple Transactions:** If there are multiple transaction with the same revision but different UUIDs, this
routine cannot reconstruct a single linear revision history. Make sure to
[clean up](crate::cleanup::cleanup_unreferenced_parquet_files) regularly to avoid this case.
This routine will fail if . Metadata Read Failure : ** There is a parquet file with metadata that cannot be read . Set ` ignore_metadata_read_failure ` to ` true ` to ignore these cases . Parquet With Revision Zero : ** One of the parquet files reports it belongs to revision ` 0 ` . This should never happen since the first transaction is always an empty one . This was likely causes by a bug or a file created by 3rd party tooling . Multiple Transactions : ** If there are multiple transaction with the same revision but different UUIDs this routine cannot reconstruct a single linear revision history . Make sure to [ clean up ] ( crate :: cleanup :: cleanup_unreferenced_parquet_files ) regularly to avoid this case .