BufferedStorageBackend

// Datastore configuration structure
type DataStoreConfig struct {
    Type   string            `toml:"type"`   // Data storage type (e.g., GCS)
    Params map[string]string `toml:"params"` // Configuration parameters for the datastore
    Schema DataStoreSchema   `toml:"schema"` // Defines the ledger storage schema
}

datastoreConfig := datastore.DataStoreConfig{
    Type: "GCS", // Using Google Cloud Storage as the backend
    Params: map[string]string{
        "destination_bucket_path": "your-gcs-bucket/data", // GCS bucket path to the data
    },
    Schema: datastore.DataStoreSchema{
        LedgersPerFile:    1,     // 1 ledger per file
        FilesPerPartition: 64000, // Number of files per partition
    },
}

// BufferedStorageBackend configuration structure
type BufferedStorageBackendConfig struct {
    BufferSize uint32        `toml:"buffer_size"`  // Number of files to buffer in memory
    NumWorkers uint32        `toml:"num_workers"`  // Number of concurrent workers for downloading ledgers
    RetryLimit uint32        `toml:"retry_limit"`  // Number of retry attempts on failure
    RetryWait  time.Duration `toml:"retry_wait"`   // Time to wait between retry attempts
}

// BufferedStorageBackend configuration instance
backendConfig := ledgerbackend.BufferedStorageBackendConfig{
    BufferSize: 100,             // Buffer upto 100 files in memory
    NumWorkers: 10,              // 10 parallel download workers
    RetryLimit: 3,               // Retry up to 3 times on failure
    RetryWait:  5 * time.Second, // Wait 5 seconds between retries

}

package main

import (
	"context"
	"log"
	"time"

	"github.com/stellar/go-stellar-sdk/ingest/ledgerbackend"
	"github.com/stellar/go-stellar-sdk/support/datastore"
	"github.com/stellar/go-stellar-sdk/support/errors"
)

func main() {
	ctx := context.Background()

	// Configure the datastore
	datastoreConfig := datastore.DataStoreConfig{
		Type: "GCS", // Google Cloud Storage as the backend
		Params: map[string]string{
			"destination_bucket_path": "your-gcs-bucket/data", // Replace with actual GCS bucket path
		},
	}

	// Initialize the datastore
	dataStore, err := datastore.NewDataStore(ctx, datastoreConfig)
	if err != nil {
		log.Fatal(errors.Wrap(err, "failed to create datastore"))
	}
	defer dataStore.Close()

	// Configure the BufferedStorageBackend
	backendConfig := ledgerbackend.BufferedStorageBackendConfig{
		BufferSize: 100,             // Number of files to buffer in memory
		NumWorkers: 10,              // Concurrent download workers
		RetryLimit: 3,               // Maximum retry attempts on failure
		RetryWait:  5 * time.Second, // Wait time between retries
	}

	// Per SEP-54 (https://github.com/stellar/stellar-protocol/blob/master/ecosystem/sep-0054.md),
	// the schema is stored in the datastore; LoadSchema retrieves it.
	schema, err := datastore.LoadSchema(context.Background(), dataStore, datastoreConfig)
	if err != nil {
		log.Fatal(errors.Wrap(err, "failed to retrieve datastore schema"))
	}

	// Initialize the backend
	backend, err := ledgerbackend.NewBufferedStorageBackend(backendConfig, dataStore, schema)
	if err != nil {
		log.Fatal(errors.Wrap(err, "failed to create buffered storage backend"))
	}
	defer backend.Close()

	// Define the ledger range to process
	ledgerRange := ledgerbackend.BoundedRange(1000, 2000)

	log.Printf("Starting ledger retrieval for range: %d - %d", ledgerRange.From(), ledgerRange.To())

	// Iterate through the ledger sequence
	for ledgerSeq := ledgerRange.From(); ledgerSeq <= ledgerRange.To(); ledgerSeq++ {
		ledgerCloseMeta, err := backend.GetLedger(ctx, ledgerSeq)
		if err != nil {
			log.Printf("Warning: Failed to retrieve ledger %d: %v", ledgerSeq, err)
			continue
		}

		log.Printf("Successfully retrieved ledger %d. Ledger sequence: %d", ledgerSeq, ledgerCloseMeta.LedgerSequence())

		// Add your logic to process the XDR data
		// Example: Parsing transactions, operations, etc.
	}

	log.Println("Ledger retrieval process completed successfully.")
}