Index

_

__post_init__() (extral.config.Config method) (extral.config.FileItemConfig method) (extral.validation.PipelineValidationResult method) (extral.validation.ValidationReport method)

A

add_error() (extral.error_tracking.ErrorReport method)

APPEND (extral.config.LoadStrategy attribute)

B

batch_size (extral.config.ExtractConfig attribute) (extral.config.FileItemConfig attribute) (extral.config.LoadConfig attribute) (extral.config.TableConfig attribute)

C

charset (extral.config.DatabaseConfig attribute) compress_data() (in module extral.store) Config (class in extral.config) config_result (extral.validation.PipelineValidationResult attribute) ConfigurationException connect() (extral.connectors.database.generic.DatabaseConnector method) (extral.connectors.database.mysql.MySQLConnector method) (extral.connectors.database.postgresql.PostgreSQLConnector method)

ConnectionException connectivity_result (extral.validation.PipelineValidationResult attribute) Connector (class in extral.connectors.connector) create_table() (extral.connectors.database.generic.DatabaseConnector method) (extral.connectors.database.mysql.MySQLConnector method) (extral.connectors.database.postgresql.PostgreSQLConnector method) CSVConnector (class in extral.connectors.file.csv_connector) CustomEncoder (class in extral.encoder)

D

database (extral.config.DatabaseConfig attribute) DatabaseConfig (class in extral.config) DatabaseConnector (class in extral.connectors.database.generic) DatabaseSchema (class in extral.schema) DatabaseTypeTranslator (class in extral.database) dataset (extral.error_tracking.ErrorDetails attribute) dataset_exists() (extral.connectors.database.generic.DatabaseConnector method) datasets (extral.state.PipelineState attribute)

DatasetState (class in extral.state) decompress_data() (in module extral.store) default() (extral.encoder.CustomEncoder method) destination (extral.config.PipelineConfig attribute) details (extral.validation.ValidationResult attribute) disconnect() (extral.connectors.database.generic.DatabaseConnector method) (extral.connectors.database.mysql.MySQLConnector method) (extral.connectors.database.postgresql.PostgreSQLConnector method) duration_seconds (extral.error_tracking.ErrorDetails attribute)

E

encode_data() (in module extral.encoder) EncodingException end_time (extral.error_tracking.ErrorReport attribute) error_message (extral.error_tracking.ErrorDetails attribute) (extral.validation.ValidationResult attribute) error_type (extral.error_tracking.ErrorDetails attribute) ErrorDetails (class in extral.error_tracking) ErrorReport (class in extral.error_tracking) errors (extral.error_tracking.ErrorReport attribute) ErrorTracker (class in extral.error_tracking) estimate_file_size() (in module extral.connectors.file.utils) extract_data() (extral.connectors.connector.Connector method) (extral.connectors.database.generic.DatabaseConnector method) (extral.connectors.database.mysql.MySQLConnector method) (extral.connectors.database.postgresql.PostgreSQLConnector method) (extral.connectors.file.base.FileConnector method) (extral.connectors.file.csv_connector.CSVConnector method) (extral.connectors.file.json_connector.JSONConnector method) extract_schema_for_table() (extral.connectors.database.generic.DatabaseConnector method) (extral.connectors.database.mysql.MySQLConnector method) (extral.connectors.database.postgresql.PostgreSQLConnector method) extract_schema_from_source() (in module extral.extract) extract_table() (in module extral.extract) extract_type (extral.config.ExtractConfig attribute) ExtractConfig (class in extral.config) ExtractException extral.config module extral.connectors.connector module extral.connectors.database.generic module extral.connectors.database.mysql module

extral.connectors.database.postgresql module extral.connectors.file.base module extral.connectors.file.csv_connector module extral.connectors.file.json_connector module extral.connectors.file.utils module extral.database module extral.encoder module extral.error_tracking module extral.exceptions module extral.extract module extral.load module extral.main module extral.schema module extral.state module extral.store module extral.validation module ExtralException

F

failed_datasets (extral.error_tracking.ErrorReport attribute) failed_pipelines (extral.error_tracking.ErrorReport attribute) field (extral.config.IncrementalConfig attribute) file_path (extral.config.FileItemConfig attribute) FileConfig (class in extral.config) FileConnector (class in extral.connectors.file.base) FileItemConfig (class in extral.config) files (extral.config.FileConfig attribute) finalize_report() (extral.error_tracking.ErrorTracker method) format (extral.config.FileItemConfig attribute)

format_validation_report() (in module extral.validation) from_dict() (extral.config.Config class method) (extral.config.DatabaseConfig class method) (extral.config.FileConfig class method) (extral.config.FileItemConfig class method) (extral.config.IncrementalConfig class method) (extral.config.LoggingConfig class method) (extral.config.PipelineConfig class method) (extral.config.ProcessingConfig class method) (extral.config.ReplaceConfig class method) (extral.config.TableConfig class method)

G

get_dataset_schema() (extral.connectors.database.generic.DatabaseConnector method) get_dataset_state() (extral.state.State method) get_effective_path() (extral.connectors.file.base.FileConnector method) get_errors_for_dataset() (extral.error_tracking.ErrorTracker method) get_errors_for_pipeline() (extral.error_tracking.ErrorTracker method)

get_file_handle() (in module extral.connectors.file.utils) get_pipeline_state() (extral.state.State method) get_summary() (extral.error_tracking.ErrorReport method) get_supported_types() (extral.database.DatabaseTypeTranslator method) get_table_schema() (extral.connectors.database.generic.DatabaseConnector method) global_conflicts (extral.validation.ValidationReport attribute)

H

has_errors_for_dataset() (extral.error_tracking.ErrorTracker method) has_errors_for_pipeline() (extral.error_tracking.ErrorTracker method)

host (extral.config.DatabaseConfig attribute) how (extral.config.ReplaceConfig attribute) http_path (extral.config.FileItemConfig attribute)

I

incremental (extral.config.TableConfig attribute) (extral.state.DatasetState attribute) incremental_field (extral.config.ExtractConfig attribute) IncrementalConfig (class in extral.config) infer_schema() (extral.connectors.file.base.FileConnector method) (extral.connectors.file.csv_connector.CSVConnector method) (extral.connectors.file.json_connector.JSONConnector method) (in module extral.schema)

initial_value (extral.config.IncrementalConfig attribute) is_http_path() (extral.connectors.file.base.FileConnector method) is_table_exists() (extral.connectors.database.generic.DatabaseConnector method) (extral.connectors.database.mysql.MySQLConnector method) (extral.connectors.database.postgresql.PostgreSQLConnector method) is_valid (extral.validation.ValidationResult attribute)

J

JSONConnector (class in extral.connectors.file.json_connector)

L

last_value (extral.config.ExtractConfig attribute) level (extral.config.LoggingConfig attribute) list_datasets() (extral.state.State method) list_pipelines() (extral.state.State method) load_data() (extral.connectors.connector.Connector method) (extral.connectors.database.generic.DatabaseConnector method) (extral.connectors.file.base.FileConnector method) (extral.connectors.file.csv_connector.CSVConnector method) (extral.connectors.file.json_connector.JSONConnector method) (in module extral.load)

load_state() (extral.state.State method) LoadConfig (class in extral.config) LoadException LoadStrategy (class in extral.config) logging (extral.config.Config attribute) LoggingConfig (class in extral.config)

M

main() (in module extral.main) MERGE (extral.config.LoadStrategy attribute) merge_key (extral.config.FileItemConfig attribute) (extral.config.LoadConfig attribute) (extral.config.TableConfig attribute) module extral.config extral.connectors.connector extral.connectors.database.generic extral.connectors.database.mysql extral.connectors.database.postgresql extral.connectors.file.base extral.connectors.file.csv_connector extral.connectors.file.json_connector extral.connectors.file.utils extral.database extral.encoder extral.error_tracking extral.exceptions extral.extract extral.load extral.main extral.schema extral.state extral.store extral.validation

mysql_to_postgresql() (in module extral.database) MySQLConnector (class in extral.connectors.database.mysql)

N

name (extral.config.FileItemConfig attribute) (extral.config.PipelineConfig attribute) (extral.config.TableConfig attribute)

O

operation (extral.error_tracking.ErrorDetails attribute) options (extral.config.FileItemConfig attribute)

overall_valid (extral.validation.PipelineValidationResult attribute) (extral.validation.ValidationReport attribute)

P

password (extral.config.DatabaseConfig attribute) pipeline (extral.error_tracking.ErrorDetails attribute) pipeline_name (extral.validation.PipelineValidationResult attribute) pipeline_results (extral.validation.ValidationReport attribute) PipelineConfig (class in extral.config) pipelines (extral.config.Config attribute) PipelineState (class in extral.state)

PipelineValidationResult (class in extral.validation) PipelineValidator (class in extral.validation) port (extral.config.DatabaseConfig attribute) postgresql_to_mysql() (in module extral.database) PostgreSQLConnector (class in extral.connectors.database.postgresql) process_table() (in module extral.main) processing (extral.config.Config attribute) ProcessingConfig (class in extral.config)

R

read_config() (extral.config.Config class method) records_processed (extral.error_tracking.ErrorDetails attribute) RECREATE (extral.config.ReplaceMethod attribute) REPLACE (extral.config.LoadStrategy attribute) replace (extral.config.TableConfig attribute) replace_method (extral.config.LoadConfig attribute)

ReplaceConfig (class in extral.config) ReplaceMethod (class in extral.config) resource_result (extral.validation.PipelineValidationResult attribute) retry_count (extral.error_tracking.ErrorDetails attribute) RetryableException run() (in module extral.main)

S

save_to_file() (extral.error_tracking.ErrorReport method) schema (extral.config.DatabaseConfig attribute) (extral.schema.DatabaseSchema attribute) (extral.schema.TargetDatabaseSchema attribute) schema_source (extral.schema.DatabaseSchema attribute) (extral.schema.TargetDatabaseSchema attribute) SchemaCreateException set_dataset_state() (extral.state.State method) source (extral.config.PipelineConfig attribute) stack_trace (extral.error_tracking.ErrorDetails attribute)

start_time (extral.error_tracking.ErrorReport attribute) State (class in extral.state) StateException store_data() (in module extral.store) store_state() (extral.state.State method) strategy (extral.config.FileItemConfig attribute) (extral.config.LoadConfig attribute) (extral.config.TableConfig attribute) successful_datasets (extral.error_tracking.ErrorReport attribute) successful_pipelines (extral.error_tracking.ErrorReport attribute) summary (extral.validation.ValidationReport attribute)

T

TableConfig (class in extral.config) tables (extral.config.DatabaseConfig attribute) TargetDatabaseSchema (class in extral.schema) test_connection() (extral.connectors.connector.Connector method) (extral.connectors.database.mysql.MySQLConnector method) (extral.connectors.database.postgresql.PostgreSQLConnector method) (extral.connectors.file.base.FileConnector method) timestamp (extral.error_tracking.ErrorDetails attribute) to_dict() (extral.config.ExtractConfig method) (extral.error_tracking.ErrorReport method) total_datasets (extral.error_tracking.ErrorReport attribute)

total_pipelines (extral.error_tracking.ErrorReport attribute) track_error() (extral.error_tracking.ErrorTracker method) translate() (extral.database.DatabaseTypeTranslator method) translate_type() (in module extral.database) TRUNCATE (extral.config.ReplaceMethod attribute) truncate_table() (extral.connectors.database.generic.DatabaseConnector method) (extral.connectors.database.mysql.MySQLConnector method) (extral.connectors.database.postgresql.PostgreSQLConnector method) type (extral.config.DatabaseConfig attribute) (extral.config.FileConfig attribute) (extral.config.IncrementalConfig attribute)

U

user (extral.config.DatabaseConfig attribute)

V

validate_configuration() (extral.validation.PipelineValidator method) ValidationException

ValidationReport (class in extral.validation) ValidationResult (class in extral.validation)

W

warnings (extral.validation.ValidationResult attribute)

workers (extral.config.PipelineConfig attribute) (extral.config.ProcessingConfig attribute)