Hi,
I have code below
http://import lseg.data as ld
import pandas as pd
import time
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
# ------------------------------------------------------------------------------
# 1. Open LSEG Data session
# ------------------------------------------------------------------------------
print("Opening LSEG Data session...")
ld.open_session()
print("Session opened.\n")
# ------------------------------------------------------------------------------
# 2. Define RICs and requested fields
# ------------------------------------------------------------------------------
continuation_rics = [
"CTc1", "CTc2", "CTc3", "CTc4", "CTc5",
"CTc6", "CTc7", "CTc8", "CTc9", "CTc10"
]
requested_fields = [
]
# ------------------------------------------------------------------------------
# 3. Define date range and chunking logic
# ------------------------------------------------------------------------------
start_date_str = "2000-01-01"
end_date_str = "2024-03-02"
start_date = datetime.strptime(start_date_str, "%Y-%m-%d")
end_date = datetime.strptime(end_date_str, "%Y-%m-%d")
print(f"Date range set from {start_date_str} to {end_date_str}.\n")
chunk_years = 1 # Chunk size: 1 year at a time
current_start = start_date
all_chunks = []
# ------------------------------------------------------------------------------
# 4. Fetch data in chunks and log timing
# ------------------------------------------------------------------------------
while current_start <= end_date:
# Determine chunk end (inclusive)
chunk_end = current_start + relativedelta(years=chunk_years) - timedelta(days=1)
if chunk_end > end_date:
chunk_end = end_date
chunk_start_str = current_start.strftime("%Y-%m-%d")
chunk_end_str = chunk_end.strftime("%Y-%m-%d")
print(f"Fetching data from {chunk_start_str} to {chunk_end_str}...")
chunk_start_time = time.time()
df_chunk = ld.get_history(
continuation_rics,
requested_fields,
start=chunk_start_str,
end=chunk_end_str,
interval="1d" # Assuming daily data intervals; adjust if needed.
)
chunk_end_time = time.time()
print(f"Chunk fetched in {chunk_end_time - chunk_start_time:.2f} seconds.\n")
if df_chunk is not None and not df_chunk.empty:
all_chunks.append(df_chunk)
else:
print("No data returned for this chunk.")
# Move to the next chunk (day after the current chunk_end)
current_start = chunk_end + timedelta(days=1)
# ------------------------------------------------------------------------------
# 5. Combine the chunks and flatten multi-index columns if necessary
# ------------------------------------------------------------------------------
if all_chunks:
full_df = pd.concat(all_chunks)
full_df.drop_duplicates(inplace=True)
print(f"Combined data shape after concatenating chunks: {full_df.shape}")
else:
full_df = pd.DataFrame()
print("No data retrieved from any chunks.")
if full_df.empty:
print("No data in the final DataFrame. Exiting.")
else:
# If the DataFrame has multi-index columns, flatten them
if isinstance(full_df.columns, pd.MultiIndex):
full_df.columns = [
f"{col[0]}_{col[1]}" if col[1] else str(col[0])
for col in full_df.columns
]
print("Flattened multi-index columns.\n")
# ------------------------------------------------------------------------------
# 6. Log missing fields for specific instruments (CTc3 to CTc10)
# ------------------------------------------------------------------------------
# Check for each instrument whether CRT_MNTH and EXPIR_DATE are present.
missing_info_instruments = []
for ric in continuation_rics:
col_crt = f"CRT_MNTH_{ric}"
col_exp = f"EXPIR_DATE_{ric}"
if col_crt not in full_df.columns or col_exp not in full_df.columns:
missing_info_instruments.append(ric)
if missing_info_instruments:
print("Note: The following instruments did not return data for CRT_MNTH and/or EXPIR_DATE:")
print(missing_info_instruments)
else:
print("All instruments returned CRT_MNTH and EXPIR_DATE data.\n")
My issue is that some data such as CRT_MNTH and EXPIR_DATE is failing to come in prior to about 2022 for all Continous contracts before 2022, what do I need to change in the code?
Could please provide the fix?
