diff --git a/src/quantlib_st/core/pandas/frequency.py b/src/quantlib_st/core/pandas/frequency.py index 65935f9..c7c164a 100644 --- a/src/quantlib_st/core/pandas/frequency.py +++ b/src/quantlib_st/core/pandas/frequency.py @@ -16,6 +16,30 @@ ) +def closing_date_rows_in_pd_object( + pd_object: Union[pd.DataFrame, pd.Series], + closing_time: pd.DateOffset = NOTIONAL_CLOSING_TIME_AS_PD_OFFSET, +) -> Union[pd.DataFrame, pd.Series]: + """ + >>> import datetime + >>> d = datetime.datetime + >>> date_index = [d(2000,1,1,15),d(2000,1,1,23), d(2000,1,2,15)] + >>> df = pd.DataFrame(dict(a=[1, 2, 3], b=[4 , 6, 5]), index=date_index) + >>> closing_date_rows_in_pd_object(df) + a b + 2000-01-01 23:00:00 2 6 + + """ + return pd_object[ + [ + check_time_matches_closing_time_to_second( + index_entry=index_entry, closing_time=closing_time + ) + for index_entry in pd_object.index + ] + ] + + def resample_prices_to_business_day_index(x: pd.Series) -> pd.Series: """Resample prices to a business-day index by taking the last available price. @@ -126,6 +150,49 @@ def infer_frequency( BUSINESS_CALENDAR_FRACTION = CALENDAR_DAYS_IN_YEAR / BUSINESS_DAYS_IN_YEAR +def sumup_business_days_over_pd_series_without_double_counting_of_closing_data( + pd_series: pd.Series, + closing_time: pd.DateOffset = NOTIONAL_CLOSING_TIME_AS_PD_OFFSET, +) -> pd.Series: + """ + Used for volume data - adds up a series over a day to get a daily total + + Uses closing values when available, otherwise sums up intraday values + + >>> import datetime + >>> d = datetime.datetime + >>> date_index1 = [d(2000,2,1,15),d(2000,2,1,16), d(2000,2,1,23), ] + >>> s1 = pd.Series([10,5,17], index=date_index1) + >>> sumup_business_days_over_pd_series_without_double_counting_of_closing_data(s1) + 2000-02-01 17 + Freq: B, Name: 0, dtype: int64 + >>> date_index1 = [d(2000,2,1,15),d(2000,2,1,16), d(2000,2,2,23) ] + >>> s1 = pd.Series([10,5,2], index=date_index1) + >>> sumup_business_days_over_pd_series_without_double_counting_of_closing_data(s1) + 2000-02-01 15.0 + 2000-02-02 2.0 + Freq: B, Name: 0, dtype: float64 + """ + intraday_data = intraday_date_rows_in_pd_object( + pd_series, closing_time=closing_time + ) + if len(intraday_data) == 0: + return pd_series + + intraday_data_summed = intraday_data.resample("1B").sum() + intraday_data_summed.name = "intraday" + + closing_data = closing_date_rows_in_pd_object(pd_series) + closing_data_summed = closing_data.resample("1B").sum() + + both_sets_of_data = pd.concat([intraday_data_summed, closing_data_summed], axis=1) + both_sets_of_data[both_sets_of_data == 0] = np.nan + joint_data = both_sets_of_data.ffill(axis=1) + joint_data = joint_data.iloc[:, 1] + + return joint_data + + def infer_frequency_approx(df_or_ts: Union[pd.DataFrame, pd.Series]) -> Frequency: avg_time_delta_in_days = average_time_delta_for_time_series(df_or_ts) diff --git a/src/quantlib_st/data/futures/csvconfig/rollconfig.csv b/src/quantlib_st/data/futures/csvconfig/rollconfig.csv new file mode 100755 index 0000000..0121322 --- /dev/null +++ b/src/quantlib_st/data/futures/csvconfig/rollconfig.csv @@ -0,0 +1,515 @@ +Instrument,HoldRollCycle,RollOffsetDays,CarryOffset,PricedRollCycle,ExpiryOffset +30YRCONF,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,11 +30YRJUMBO,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,9 +AEX,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,19 +AEX_mini,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,19 +ALUMINIUM,FGHJKMNQUVXZ,-45,1,FGHJKMNQUVXZ,14 +ALUMINIUM_LME,FGHJKMNQUVXZ,-45,1,FGHJKMNQUVXZ,14 +AMERIBOR-1M,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,26 +AMERIBOR-3M,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,26 +AMERIBOR-T30,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,26 +AMX,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,14 +ASX,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,18 +ATX,HMUZ,-5,1,HMUZ,16 +AUD,HMUZ,-5,1,HMUZ,17 +AUD-ICE,HMUZ,-5,1,HMUZ,16 +AUD-SGX,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,12 +AUDCAD,HMUZ,-5,1,HMUZ,16 +AUDJPY,HMUZ,-5,1,HMUZ,17 +AUDJPY-SGX,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,13 +AUD_micro,HMUZ,-5,1,HMUZ,17 +AUSCASH,FGHJKMNQUVXZ,-20,1,FGHJKMNQUVXZ,30 +BARLEY,FHKNUX,-5,1,FHKNUX,14 +BB3M,HMUZ,-90,-1,HMUZ,15 +BBCOMM,HMUZ,-5,1,HMUZ,15 +BEL20,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,15 +BITCOIN,FGHJKMNQUVXZ,-4,1,FGHJKMNQUVXZ,26 +BITCOIN-BAKKT,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,28 +BOBL,HMUZ,-5,1,HMUZ,6 +BONO,HMUZ,-5,1,HMUZ,6 +BONO-MEFF,HMUZ,-5,1,HMUZ,13 +BONO10,HMUZ,-5,1,HMUZ,28 +BONO3,HMUZ,-5,1,HMUZ,25 +BOVESPA,GJMQVZ,-5,1,GJMQVZ,15 +BRE,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,17 +BRENT,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,19 +BRENT-LAST,FGHJKMNQUVXZ,-40,-1,FGHJKMNQUVXZ,-32 +BRENT_W,FGHJKMNQUVXZ,-40,-1,FGHJKMNQUVXZ,14 +BRR,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,26 +BRREUR,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,21 +BTP,HMUZ,-5,1,HMUZ,6 +BTP3,HMUZ,-5,1,HMUZ,10 +BTP5,HMUZ,-5,1,HMUZ,10 +BUND,HMUZ,-5,1,HMUZ,6 +BUTTER,FGHJKMNQUVXZ,-40,-1,FGHJKMNQUVXZ,15 +BUXL,HMUZ,-5,1,HMUZ,6 +CAC,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,20 +CAD,HMUZ,-5,1,HMUZ,17 +CAD10,HMUZ,-20,1,HMUZ,20 +CAD2,HMUZ,-20,1,HMUZ,20 +CAD5,HMUZ,-20,1,HMUZ,20 +CADJPY,HMUZ,-5,1,HMUZ,17 +CADJPY2,HMUZ,-5,1,HMUZ,16 +CADSTIR,HMUZ,-200,-1,FGHJKMNQUVXZ,15 +CAD_micro,HMUZ,-5,1,HMUZ,17 +CAN-ENERGY,HMUZ,-5,1,HMUZ,16 +CAN-FINANCE,HMUZ,-5,1,HMUZ,15 +CAN-GOLD,HMUZ,-5,1,HMUZ,16 +CAN-TECH,HMUZ,-5,1,HMUZ,16 +CANOLA,FHKNX,-60,-1,FHKNX,13 +CETES,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,13 +CH10,HMUZ,-5,1,HMUZ,6 +CHEESE,FGHJKMNQUVXZ,-40,-1,FGHJKMNQUVXZ,15 +CHF,HMUZ,-5,1,HMUZ,17 +CHFJPY,HMUZ,-5,1,HMUZ,17 +CHFJPY-ICE,HMUZ,-5,1,HMUZ,16 +CHF_micro,HMUZ,-5,1,HMUZ,17 +CHINA120,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,28 +CHINAA-CON,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,18 +CLP,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,1 +CNH,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,15 +CNH-CME,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,19 +CNH-CME_micro,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,19 +CNH-HK,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,17 +CNH-onshore,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,15 +CNHEUR,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,19 +COAL,FGHJKMNQUVXZ,-45,-1,FGHJKMNQUVXZ,24 +COAL-GEORDIE,FGHJKMNQUVXZ,-45,-1,FGHJKMNQUVXZ,27 +COAL-RICH-BAY,FGHJKMNQUVXZ,-90,-1,FGHJKMNQUVXZ,27 +COCOA,HKNUZ,-50,-1,HKNUZ,13 +COCOA_LDN,HKNUZ,-50,-1,HKNUZ,13 +COFFEE,HKNUZ,-50,-1,HKNUZ,19 +COPPER,HNUZ,-30,1,FHJMNUVZ,27 +COPPER-micro,HKNUZ,-30,1,FGHJKMNQUVXZ,15 +COPPER-mini,HKNUZ,-30,1,HKNUZ,27 +COPPER_LME,FGHJKMNQUVXZ,-45,1,FGHJKMNQUVXZ,14 +CORN,Z,-60,-1,HKNUZ,14 +CORN-EURO,HMQX,-35,-1,HMQX,6 +CORN-JPN,FHKNUX,-5,1,FHKNUX,21 +CORN_mini,HKNUZ,-30,1,HKNUZ,14 +CORRA,HMUZ,-180,-1,HMUZ,19 +COTTON,HKNVZ,-5,1,HKNVZ,15 +COTTON2,HKNZ,-60,-1,HKNVZ,6 +CRUDE_ICE,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,16 +CRUDE_W,Z,-40,-1,FGHJKMNQUVXZ,-11 +CRUDE_W_micro,FGHJKMNQUVXZ,-40,-1,FGHJKMNQUVXZ,-11 +CRUDE_W_mini,FGHJKMNQUVXZ,-40,-1,FGHJKMNQUVXZ,-11 +CZK,HMUZ,-5,1,HMUZ,15 +DAX,HMUZ,-5,1,HMUZ,15 +DIVDAX,HMUZ,-5,1,HMUZ,19 +DIVDAX-DIVI,HMUZ,-5,1,HMUZ,15 +DIVDAX-DIVI2,Z,-5,1,Z,20 +DJSTX-SMALL,HMUZ,-10,1,HMUZ,18 +DJUBS,FGHJMNQUVXZ,-5,1,FGHJMNQUVXZ,18 +DOW,HMUZ,-5,1,HMUZ,15 +DOW_YEN,HMUZ,-5,1,HMUZ,17 +DOW_mini,HMUZ,-5,1,HMUZ,15 +DX,HMUZ,-30,1,HMUZ,13 +EDOLLAR,HMUZ,-1100,-1,HMUZ,18 +EPRA-EURO,HMU,-5,1,HMU,19 +EPRA-EUROPE,HMUZ,-5,1,HMUZ,13 +ETHANOL,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,-11 +ETHER-micro,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,15 +ETHEREUM,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,15 +ETHRR,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,20 +ETHRREUR,FGHJKMNUXZ,-5,1,FGHJKMNUXZ,20 +EU-AUTO,HMUZ,-5,1,HMUZ,15 +EU-BANKS,HMUZ,-10,1,HMUZ,18 +EU-BANKS-DIVI,MZ,-5,1,MZ,19 +EU-BANKS2,HMUZ,-5,1,HMUZ,20 +EU-BASIC,HMUZ,-5,1,HMUZ,15 +EU-CHEM,HMUZ,-10,1,HMUZ,18 +EU-CONSTRUCTION,HMUZ,-10,1,HMUZ,18 +EU-DIV30,HMUZ,-5,1,HMUZ,15 +EU-DIV30-DVP,Z,-5,1,Z,19 +EU-DIV50,MZ,-60,1,MZ,15 +EU-DJ-AUTO,HMUZ,-5,1,HMUZ,14 +EU-DJ-BASIC,HMUZ,-5,1,HMUZ,16 +EU-DJ-CHEM,HMUZ,-5,1,HMUZ,17 +EU-DJ-CONSTRUCTION,HMUZ,-5,1,HMUZ,18 +EU-DJ-FINANCE,HMUZ,-5,1,HMUZ,19 +EU-DJ-FOOD,HMUZ,-5,1,HMUZ,15 +EU-DJ-HEALTH,HMUZ,-5,1,HMUZ,14 +EU-DJ-HOUSE,HMUZ,-5,1,HMUZ,18 +EU-DJ-INDUSTRY,HMUZ,-5,1,HMUZ,20 +EU-DJ-INDUSTRY2,HMUZ,-5,1,HMUZ,14 +EU-DJ-INSURE,HMUZ,-5,1,HMUZ,17 +EU-DJ-MEDIA,HMUZ,-5,1,HMUZ,17 +EU-DJ-OIL,HMUZ,-5,1,HMUZ,15 +EU-DJ-RETAIL,HMUZ,-5,1,HMUZ,19 +EU-DJ-TECH,HMUZ,-5,1,HMUZ,15 +EU-DJ-TELECOM,HMUZ,-5,1,HMUZ,15 +EU-DJ-TRAVEL,HMUZ,-5,1,HMUZ,16 +EU-DJ-UTIL,HMUZ,-5,1,HMUZ,15 +EU-ESG,HMUZ,-5,1,HMUZ,18 +EU-FINANCE,HMUZ,-5,1,HMUZ,17 +EU-FOOD,HMUZ,-5,1,HMUZ,15 +EU-HEALTH,HMUZ,-5,1,HMUZ,15 +EU-HOUSE,HMUZ,-5,1,HMUZ,15 +EU-INSURE,HMUZ,-10,1,HMUZ,18 +EU-MEDIA,HMUZ,-5,1,HMUZ,15 +EU-MID,HMUZ,-5,1,HMUZ,15 +EU-OIL,HMUZ,-5,1,HMUZ,15 +EU-REALESTATE,HMUZ,-5,1,HMUZ,15 +EU-RETAIL,HMUZ,-5,1,HMUZ,15 +EU-TECH,HMUZ,-5,1,HMUZ,15 +EU-TELECOM,HMUZ,-5,1,HMUZ,19 +EU-TRAVEL,HMUZ,-5,1,HMUZ,15 +EU-UTILS,HMUZ,-5,1,HMUZ,15 +EUA,Z,-270,-1,HMUZ,14 +EUIRS10,HMUZ,-5,1,HMUZ,16 +EUIRS2,HMUZ,-5,1,HMUZ,15 +EUIRS5,HMUZ,-5,1,HMUZ,15 +EUR,HMUZ,-5,1,HMUZ,17 +EUR-ICE,HMUZ,-5,1,HMUZ,17 +EURAUD,HMUZ,-5,1,HMUZ,17 +EURAUD-ICE,HMUZ,-5,1,HMUZ,17 +EURCAD,HMUZ,-5,1,HMUZ,15 +EURCAD-ICE,HMUZ,-5,1,HMUZ,17 +EURCHF,HMUZ,-5,1,HMUZ,17 +EURCHF-ICE,HMUZ,-5,1,HMUZ,20 +EURCZK,HMUZ,-5,1,HMUZ,20 +EURHUF,HMUZ,-5,1,HMUZ,17 +EURIBOR,HMUZ,-700,-1,HMUZ,15 +EURIBOR-ICE,HMUZ,-700,-1,HMUZ,15 +EURINR,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,18 +EURMXP,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,15 +EURO600,HMUZ,-5,1,HMUZ,15 +EURO600-ESG,HMUZ,-5,1,HMUZ,21 +EUROFIRST100,HMU,-5,1,HMU,19 +EUROFIRST80,HMU,-5,1,HMU,19 +EUROSTX,HMUZ,-5,1,HMUZ,15 +EUROSTX-CORE,HMUZ,-5,1,HMUZ,20 +EUROSTX-DJ,HMUZ,-5,1,HMUZ,21 +EUROSTX-LARGE,HMUZ,-5,1,HMUZ,15 +EUROSTX-MID,HMUZ,-5,1,HMUZ,18 +EUROSTX-PRICE,HMUZ,-5,1,HMUZ,19 +EUROSTX-SMALL,HMUZ,-5,1,HMUZ,15 +EUROSTX200-LARGE,HMUZ,-5,1,HMUZ,15 +EUR_micro,HMUZ,-5,1,HMUZ,17 +EUR_mini,HMUZ,-5,1,HMUZ,17 +FANG,HMUZ,-5,1,HMUZ,16 +FED,FGHJKMNQUVXZ,-180,-1,FGHJKMNQUVXZ,15 +FEEDCOW,FHJKQUVX,-90,-1,FHJKQUVX,15 +FTSE100,HMUZ,-30,1,HMUZ,17 +FTSE100-DIV,HMUZ,-30,1,HMUZ,17 +FTSE250,HMUZ,-5,1,HMUZ,15 +FTSECHINAA,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,15 +FTSECHINAA-CSOP,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,28 +FTSECHINAA-IS,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,25 +FTSECHINAH,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,14 +FTSEINDO,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,18 +FTSETAIWAN,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,15 +FTSEVIET,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,15 +GAS-LAST,FGHJKMNQUVXZ,-35,-1,FGHJKMNQUVXZ,-5 +GAS-PEN,FGHJKMNQUVXZ,-35,-1,FGHJKMNQUVXZ,-5 +GASOIL,FGHJKMNQUVXZ,-60,-1,FGHJKMNQUVXZ,12 +GASOILINE,FGHJKMNQUVXZ,-60,-1,FGHJKMNQUVXZ,14 +GASOILINE_ICE,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,20 +GASOILINE_micro,FGHJKMNQUVXZ,-30,1,FGHJKMNQUVXZ,14 +GAS_NL,HMUZ,-60,-1,HMUZ,-4 +GAS_UK,HMUZ,-60,-1,HMUZ,-4 +GAS_US,FGHJKMNQUVXZ,-35,-1,FGHJKMNQUVXZ,-4 +GAS_US_mini,FGHJKMNQUVXZ,-35,-1,FGHJKMNQUVXZ,-4 +GBP,HMUZ,-5,1,HMUZ,17 +GBP-ICE,HMUZ,-5,1,HMUZ,15 +GBPCHF,HMUZ,-5,1,HMUZ,17 +GBPCHF-ICE,HMUZ,-5,1,HMUZ,18 +GBPEUR,HMUZ,-5,1,HMUZ,15 +GBPEUR-ICE,HMUZ,-5,1,HMUZ,15 +GBPINR,FGHJKMNQUVXZ,-20,1,FGHJKMNQUVXZ,20 +GBPJPY,HMUZ,-5,1,HMUZ,15 +GBPJPY-ICE,HMUZ,-5,1,HMUZ,16 +GBP_micro,HMUZ,-5,1,HMUZ,17 +GICS,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,15 +GICS-EXCESS,FGHJKMUXZ,-5,1,FGHJKMUXZ,18 +GILT,HMUZ,-5,1,HMUZ,28 +GILT2,HMUZ,-5,1,HMUZ,26 +GILT5,HMUZ,-5,1,HMUZ,23 +GOLD,GJMQVZ,-30,1,GJMQVZ,26 +GOLD-CHINA,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,29 +GOLD-CHINA-USD,FGHJMQVZ,-5,1,FGHJMQVZ,26 +GOLD-CN-HK,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,14 +GOLD-HK,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,17 +GOLD-JPN,GJMQVZ,-5,1,GJMQVZ,22 +GOLD-JPN_mini,GJMQVZ,-5,1,GJMQVZ,20 +GOLD-mini,GJMQVZ,-30,1,GJMQVZ,26 +GOLD_micro,GJMQVZ,-40,1,GJMQVZ,26 +HANG,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,26 +HANG-DIV,Z,-250,-1,Z,26 +HANGTECH,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,25 +HANGENT,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,28 +HANGENT-GTR,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,23 +HANGENT-NTR,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,25 +HANGENT_mini,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,28 +HANG_mini,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,23 +HEAT-DEG-AMS,FGHJVXZ,-5,1,FGHJVXZ,20 +HEAT-DEG-LON,FGHJVXZ,-5,1,FGHJVXZ,20 +HEAT-DEG-NY,FGHJVXZ,-5,1,FGHJVXZ,20 +HEATOIL,FGHJKMNQUVXZ,-60,-1,FGHJKMNQUVXZ,14 +HEATOIL-ICE,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,26 +HEATOIL-mini,FGHJKMNQUVXZ,-30,1,FGHJKMNQUVXZ,14 +HEATOIL_micro,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,24 +HIBOR,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,16 +HIGHYIELD,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,3 +HOUSE-BO,GKQX,-5,1,GKQX,19 +HOUSE-CG,GKQX,-5,1,GKQX,26 +HOUSE-DC,GKQX,-5,1,GKQX,22 +HOUSE-DN,GKQX,-5,1,GKQX,19 +HOUSE-LA,GKQX,-5,1,GKQX,23 +HOUSE-LV,GKQX,-5,1,GKQX,19 +HOUSE-MI,GKQX,-5,1,GKQX,23 +HOUSE-NY,GKQX,-5,1,GKQX,27 +HOUSE-SD,GKQX,-5,1,GKQX,25 +HOUSE-SF,GKQX,-5,1,GKQX,29 +HOUSE-US,GKQX,-5,1,GKQX,15 +HSCEI-DIV,Z,-250,-1,Z,22 +HUF,HMUZ,-5,1,HMUZ,22 +HUFEUR,HMUZ,-5,1,HMUZ,19 +IBEX,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,15 +IBEX_mini,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,15 +IG,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,3 +IND-BANK,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,15 +IND-FIN,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,15 +INR,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,15 +INR-SGX,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,15 +INR-SGX1,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,14 +INR-SGX2,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,24 +INR-micro,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,15 +IPC,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,15 +IRON,FGHJKMNQUVXZ,-60,-1,FGHJKMNQUVXZ,27 +IRON-CME,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,20 +IRON-HK,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,16 +IRS,HMUZ,-10,1,HMUZ,15 +JGB,HMUZ,-20,1,HMUZ,15 +JGB-SGX-mini,HMUZ,-20,1,HMUZ,15 +JGB-mini,HMUZ,-20,1,HMUZ,15 +JP-REALESTATE,HMUZ,-5,1,HMUZ,15 +JPY,HMUZ,-5,1,HMUZ,17 +JPY-SGX,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,15 +JPY-SGX-TITAN,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,18 +JPYINR,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,20 +JPY_micro,HMUZ,-5,1,HMUZ,20 +JPY_mini,HMUZ,-5,1,HMUZ,17 +KOSDAQ,HMUZ,-1,1,HMUZ,13 +KOSPI,HMUZ,-1,1,HMUZ,13 +KOSPI300,HMUZ,-5,1,HMUZ,20 +KOSPI_mini,FGHJKMNQUVXZ,-1,1,FGHJKMNQUVXZ,13 +KR10,HMUZ,-1,1,HMUZ,18 +KR3,HMUZ,-1,1,HMUZ,18 +KRW,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,12 +KRWJPY,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,18 +KRWUSD,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,15 +KRWUSD_mini,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,15 +LEAD_LME,FGHJKMNQUVXZ,-45,1,FGHJKMNQUVXZ,14 +LEANHOG,GJMNQVZ,-60,-1,GJKMNQVZ,13 +LIBOR1,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,15 +LIVECOW,GJMQVZ,-60,-1,GJMQVZ,28 +LUMBER,FHKNUX,-50,1,FHKNUX,15 +LUMBER-new,FHKNUX,-50,1,FHKNUX,15 +MARS-ARGUS,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,19 +MIB,HMUZ,-5,1,HMUZ,20 +MIB-DIVI,MZ,-5,1,MZ,19 +MIB_micro,HMUZ,-5,1,HMUZ,18 +MIB_mini,HMUZ,-5,1,HMUZ,18 +MID-DAX,HMUZ,-5,1,HMUZ,15 +MILK,FGHJKMNQUVXZ,-60,-1,FGHJKMNQUVXZ,14 +MILKDRY,FGHJKMNQUVXZ,-180,-1,FGHJKMNQUVXZ,18 +MILKWET,FGHJKMNQUVXZ,-90,-1,FGHJKMNQUVXZ,15 +MILLWHEAT,HKUZ,-60,-1,HKUZ,9 +MSCIASIA,HMUZ,-5,1,HMUZ,15 +MSCIBRAZIL,HMUZ,-5,1,HMUZ,15 +MSCIEAFA,HMUZ,-5,1,HMUZ,19 +MSCIEM,HJKMUZ,-10,1,HJKMUZ,17 +MSCIEM-LIFFE,HMUZ,-5,1,HMUZ,15 +MSCIEMASIA,M,-5,1,M,17 +MSCIEURONET,HMUZ,-5,1,HMUZ,19 +MSCIEURONET-ICE,HMUZ,-5,1,HMUZ,15 +MSCIEUROPE,HJKMUZ,-10,1,HJKMUZ,17 +MSCIEUROPE-ICE,HMUZ,-5,1,HMUZ,15 +MSCIEUROPE-LIFFE,HMUZ,-5,1,HMUZ,20 +MSCIINDO,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,21 +MSCIJAPAN,HMUZ,-5,1,HMUZ,20 +MSCIJAPAN-LIFFE,HMUZ,-5,1,HMUZ,20 +MSCILATIN,HMUZ,-5,1,HMUZ,20 +MSCIPANEURO-LIFFE,HMUZ,-5,1,HMUZ,21 +MSCISING,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,27 +MSCITAIWAN,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,24 +MSCIUSA,HMUZ,-5,1,HMUZ,19 +MSCIWORLD,HMUZ,-5,1,HMUZ,15 +MSCIWORLD-MINVOL,HMUZ,-5,1,HMUZ,15 +MSCIWORLDNET-EUR,HMUZ,-5,1,HMUZ,19 +MSCIWORLDNET-USD,HMUZ,-5,1,HMUZ,19 +MUMMY,HMUZ,-5,1,HMUZ,14 +MXP,HMUZ,-5,1,HMUZ,17 +NASDAQ,HMUZ,-5,1,HMUZ,14 +NASDAQ_micro,HMUZ,-5,1,HMUZ,14 +NASDAQ_mini,HMUZ,-5,1,HMUZ,18 +NICKEL_LME,FGHJKMNQUVXZ,-45,1,FGHJKMNQUVXZ,14 +NIFTY,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,15 +NIFTY-IN,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,20 +NIKKEI,HMUZ,-5,1,FGHJKMNQUVXZ,14 +NIKKEI-CME,HMUZ,-5,1,HMUZ,19 +NIKKEI-JPY,FGHJKMNUVXZ,-10,1,FGHJKMNUVXZ,19 +NIKKEI-JPY_mini,HMUZ,-10,1,HMUZ,15 +NIKKEI-SGX,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,21 +NIKKEI-SGX-DIV,Z,-10,1,Z,20 +NIKKEI-SGX-USD,HMUZ,-10,1,HMUZ,14 +NIKKEI-SGX_mini,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,18 +NIKKEI400,HMUZ,-5,1,HMUZ,14 +NIKKEI_large,HMUZ,-5,1,HMUZ,19 +NOK,HMUZ,-5,1,HMUZ,17 +NZD,HMUZ,-10,1,HMUZ,17 +OAT,HMUZ,-5,1,HMUZ,6 +OAT5,HMUZ,-5,1,HMUZ,20 +OATIES,HKNUZ,-90,-1,HKNUZ,14 +OJ,FHKNUX,-40,1,FHKNUX,9 +OMX,HMUZ,-10,1,HMUZ,15 +OMX-SWE,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,19 +OMXESG,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,13 +OMXSB,HMUZ,-5,1,HMUZ,-14 +PALLAD,HMUZ,-30,1,HMUZ,26 +PIPELINE,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,27 +PLAT,FJNV,-30,1,FJNV,26 +PLAT-JPN,GJMQVZ,-5,1,GJMQVZ,24 +PLAT-JPN_mini,GJMQVZ,-5,1,GJMQVZ,23 +PLN,HMUZ,-10,1,HMUZ,15 +PLZEUR,HMUZ,-5,1,HMUZ,12 +R1000,HMUZ,-10,1,HMUZ,18 +R1000GROWTH-mini,HMUZ,-10,1,HMUZ,18 +R1000_mini,HMUZ,-10,1,HMUZ,18 +RAPESEED,GKQX,-60,-1,GKQX,27 +REDWHEAT,HKNUZ,-90,-1,HKNUZ,14 +RICE,FHKNUX,-90,-1,FHKNUX,14 +ROBUSTA,FHKNUX,-50,-1,FHKNUX,24 +RUBBER,FGHJKMNQUVXZ,-30,1,FGHJKMNQUVXZ,-1 +RUBBER-RSS,FGHJKMNQUVXZ,-87,1,FGHJKMNQUVXZ,24 +RUR,HMUZ,-5,1,HMUZ,17 +RUSSELL,HMUZ,-5,1,HMUZ,15 +RUSSELL-GROWTH,HMUZ,-5,1,HMUZ,17 +RUSSELL-VALUE,HMUZ,-5,1,HMUZ,17 +RUSSELL_mini,HMUZ,-5,1,HMUZ,15 +SARONA,HMUZ,-200,-1,HMUZ,19 +SEK,HMUZ,-5,1,HMUZ,17 +SGD,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,15 +SGD_mini,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,15 +SGX,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,15 +SHATZ,HMUZ,-5,1,HMUZ,6 +SILVER,HKNUZ,-45,-1,FGHJKMNQUVXZ,26 +SILVER-mini,FHKNUZ,-45,-1,FHKNUZ,-5 +SING-REALESTATE,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,23 +SMI,HMUZ,-5,1,HMUZ,14 +SMI-DIV,Z,-5,1,Z,19 +SMI-MID,HMUZ,-5,1,HMUZ,14 +SMIETF,HMUZ,-5,1,HMUZ,19 +SOFR,HMUZ,-1000,-1,HMUZ,18 +SOFR1,HMUZ,-200,-1,HMUZ,5 +SONIA,HMUZ,-5,1,HMUZ,14 +SONIA1,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,27 +SONIA3,HMUZ,-1000,-1,HMUZ,9 +SOYBEAN,X,-60,-1,FHKNQUX,13 +SOYBEAN_mini,FHKNQUX,-45,1,FHKNQUX,13 +SOYMEAL,FHKNQUVZ,-90,-1,FHKNQUVZ,14 +SOYOIL,FHKNQUVZ,-90,-1,FHKNQUVZ,14 +SP400,HMUZ,-5,1,HMUZ,15 +SP500,HMUZ,-5,1,HMUZ,14 +SP500-GROWTH,HMZ,-5,1,HMZ,19 +SP500-VALUE,HMZ,-5,1,HMZ,18 +SP500_micro,HMUZ,-5,1,HMUZ,14 +SP500_mini,HMUZ,-5,1,HMUZ,17 +SP600-SMALL,HMUZ,-5,1,HMUZ,19 +SPI200,HMUZ,-30,1,FGHJKMNQUVXZ,14 +STEEL,FGHJKMNQUVXZ,-90,-1,FGHJKMNQUVXZ,15 +STERLING3,HMUZ,-30,1,FGHJKMNQUVXZ,15 +SUGAR11,HKNV,-60,-1,HKNV,-2 +SUGAR16,FHKNUX,-60,-1,FHKNUX,7 +SUGAR_WHITE,HKQVZ,-60,-1,HKQVZ,-2 +SWISSLEAD,HMUZ,-5,1,HMUZ,14 +TECDAX,HMUZ,-5,1,HMUZ,19 +THB,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,29 +TIN_LME,FGHJKMNQUVXZ,-45,1,FGHJKMNQUVXZ,14 +TOPIX,HMUZ,-5,1,HMUZ,14 +TOPIX30,HMUZ,-5,1,HMUZ,13 +TOPIX_Large,HMUZ,-5,1,HMUZ,15 +TSE60,HMUZ,-5,1,HMUZ,14 +TSX,HMUZ,-5,1,HMUZ,16 +TWD,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,15 +TWD-mini,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,15 +UMBS-20,FGHJKZ,-5,1,FGHJKZ,15 +UMBS-25,FGHJKZ,-5,1,FGHJKZ,15 +UMBS-30,FGHJKZ,-5,1,FGHJKZ,15 +UMBS-35,FGHJKZ,-5,1,FGHJKZ,15 +UMBS-40,FGHJKZ,-5,1,FGHJKZ,15 +UMBS-45,FGHJKZ,-5,1,FGHJKZ,15 +UMBS-50,FGHJKZ,-5,1,FGHJKZ,15 +URANIUM,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,17 +US-BIOTECH,HMUZ,-5,1,HMUZ,21 +US-DISCRETE,HMUZ,-5,1,HMUZ,15 +US-ENERGY,HMUZ,-5,1,HMUZ,15 +US-FINANCE,HMUZ,-5,1,HMUZ,15 +US-HEALTH,HMUZ,-5,1,HMUZ,15 +US-INDUSTRY,HMUZ,-5,1,HMUZ,15 +US-INSURE,HMUZ,-5,1,HMUZ,20 +US-MATERIAL,HMUZ,-5,1,HMUZ,15 +US-OILGAS,HMUZ,-5,1,HMUZ,18 +US-PROPERTY,HMUZ,-5,1,HMUZ,15 +US-REALESTATE,HMUZ,-2,1,HMUZ,17 +US-REGBANK,HMUZ,-5,1,HMUZ,20 +US-RETAIL,HMUZ,-5,1,HMUZ,21 +US-SEMICONDUCTOR,HMUZ,-5,1,HMUZ,20 +US-STAPLES,HMUZ,-5,1,HMUZ,15 +US-TECH,HMUZ,-5,1,HMUZ,15 +US-UTILS,HMUZ,-5,1,HMUZ,15 +US10,HMUZ,-25,1,HMUZ,19 +US10U,HMUZ,-25,1,HMUZ,19 +US10Y_micro,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,29 +US10Y_small,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,15 +US2,HMUZ,-25,1,HMUZ,19 +US20,HMUZ,-25,1,HMUZ,19 +US20-new,HMUZ,-25,1,HMUZ,19 +US2Y_micro,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,29 +US2Y_small,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,15 +US3,HMUZ,-25,1,HMUZ,19 +US30,HMUZ,-25,1,HMUZ,19 +US30Y_micro,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,29 +US30Y_small,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,14 +US5,HMUZ,-25,1,HMUZ,19 +US5Y_micro,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,28 +USDCAD_micro,HMUZ,-5,1,HMUZ,19 +USDCHF_micro,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,15 +USDCNH-CME,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,20 +USDCNH-HK,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,14 +USDCNH-SGX_mini,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,15 +USDINR,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,15 +USDKRW,HMUZ,-10,1,HMUZ,15 +USDMXP,FGHJKMNQUVXZ,-10,1,FGHJKMNQUVXZ,15 +USIRS10,HMUZ,-20,1,HMUZ,15 +USIRS10ERIS,HMZ,-5,1,HMZ,19 +USIRS12ERIS,HMUZ,-5,1,HMUZ,-62 +USIRS15ERIS,HMUZ,-5,1,HMUZ,-58 +USIRS2,HMZ,-5,1,HMZ,19 +USIRS20ERIS,HMUZ,-5,1,HMUZ,-18 +USIRS2ERIS,HMUZ,-740,1,HMUZ,15 +USIRS30,HMZ,-5,1,HMZ,19 +USIRS3ERIS,HMUZ,-5,1,HMUZ,19 +USIRS4ERIS,HMUZ,-5,1,HMUZ,19 +USIRS5,HMUZ,-20,1,HMUZ,15 +USIRS5ERIS,HMUZ,-1835,1,HMUZ,15 +USIRS5ERIS_SOFR,HMUZ,-1835,1,HMUZ,15 +USIRS7ERIS,HMUZ,-5,1,HMUZ,19 +V2X,FGHJKMNQUVXZ,-60,-1,FGHJKMNQUVXZ,19 +VHANG,FGHJKMNQUVXZ,-5,1,FGHJKMNQUVXZ,26 +VIX,FGHJKMNQUVXZ,-60,-1,FGHJKMNQUVXZ,19 +VIX_mini,FGHJKMNQUVXZ,-60,-1,FGHJKMNQUVXZ,19 +VNKI,FGHJKMNQUVXZ,-60,-1,FGHJKMNQUVXZ,15 +VOLQ,FGHJKMNQVXZ,-5,1,FGHJKMNQVXZ,17 +WATER-CALI,FGHJKMNQUXZ,-5,1,FGHJKMNQUXZ,16 +WHEAT,Z,-60,-1,HKNUZ,13 +WHEAT-ASX,FHKNU,-5,1,FHKNU,13 +WHEAT_ICE,KX,-100,-1,FHKNX,6 +WHEAT_mini,HKNUZ,-30,1,HKNUZ,13 +WHEY,FGHJKMNQUVXZ,-40,-1,FGHJKMNQUVXZ,15 +YENEUR,HMUZ,-5,1,HMUZ,15 +YENEUR-ICE,HMUZ,-5,1,HMUZ,14 +ZAR,HMUZ,-5,1,HMUZ,12 +ZINC_LME,FGHJKMNQUVXZ,-45,1,FGHJKMNQUVXZ,14 diff --git a/src/quantlib_st/init/futures/build_multiple_prices_from_raw_data.py b/src/quantlib_st/init/futures/build_multiple_prices_from_raw_data.py index ca18507..7082f8d 100644 --- a/src/quantlib_st/init/futures/build_multiple_prices_from_raw_data.py +++ b/src/quantlib_st/init/futures/build_multiple_prices_from_raw_data.py @@ -27,6 +27,7 @@ def iterate_roll(self): self.rolling_row_index = self.rolling_row_index + 1 def not_end_of_calendar(self): + assert self.roll_calendar is not None return self.rolling_row_index < len(self.roll_calendar.index) def data_now_added(self): @@ -130,6 +131,7 @@ def _calc_roll_date_info( roll_calendar = roll_calendar_with_roll_index.roll_calendar rolling_row_index = roll_calendar_with_roll_index.rolling_row_index + assert roll_calendar is not None last_roll_date = roll_calendar.index[rolling_row_index - 1] next_roll_date = roll_calendar.index[rolling_row_index] diff --git a/src/quantlib_st/init/futures/build_roll_calendars.py b/src/quantlib_st/init/futures/build_roll_calendars.py new file mode 100644 index 0000000..44c9c73 --- /dev/null +++ b/src/quantlib_st/init/futures/build_roll_calendars.py @@ -0,0 +1,688 @@ +from collections import namedtuple +from copy import copy + +import numpy as np +import pandas as pd + +from quantlib_st.core.exceptions import missingData +from quantlib_st.objects.contract_dates_and_expiries import contractDate +from quantlib_st.objects.dict_of_futures_per_contract_prices import ( + dictFuturesContractFinalPrices, +) +from quantlib_st.objects.multiple_prices import futuresMultiplePrices +from quantlib_st.objects.roll_parameters_with_price_data import ( + find_earliest_held_contract_with_price_data, + contractWithRollParametersAndPrices, +) +from quantlib_st.objects.rolls import rollParameters, contractDateWithRollParameters + + +def generate_approximate_calendar( + roll_parameters_object: rollParameters, + dict_of_futures_contract_prices: dictFuturesContractFinalPrices, +) -> pd.DataFrame: + """ + Using a rollData object we work out roughly what the rolls should be (in an ideal world with available prices all the time) + for contracts held between start_date and end_date + + Called by __init__ + + :param dict_of_futures_contract_prices: dict, keys are contract date ids 'yyyymmdd' + :param roll_parameters_object: rollData + + :return: data frame ready to be rollCalendar + """ + try: + earliest_contract_with_roll_data = find_earliest_held_contract_with_price_data( + roll_parameters_object, dict_of_futures_contract_prices + ) + except missingData: + raise Exception("Can't find any valid starting contract!") + + approx_calendar = _create_approx_calendar_from_earliest_contract( + earliest_contract_with_roll_data, + ) + + return approx_calendar + + +INDEX_NAME = "current_roll_date" + + +class _rollCalendarRow(dict): + def __init__( + self, + current_roll_date, + current_contract: str, + next_contract: str, + carry_contract: str, + ): + # a dict because pd.DataFrame can handle those + # plus a hidden storage of the actual contract + + super().__init__({}) + if current_roll_date is not None: + self[INDEX_NAME] = current_roll_date + self["current_contract"] = current_contract + self["next_contract"] = next_contract + self["carry_contract"] = carry_contract + + @property + def roll_date(self): + return self[INDEX_NAME] + + +_bad_row = _rollCalendarRow(None, None, None, None) + + +class _listOfRollCalendarRows(list): + def to_pd_df(self): + result = pd.DataFrame(self) + result.index = result[INDEX_NAME] + result = result.drop(labels=INDEX_NAME, axis=1) + + return result + + def last_roll_date(self): + last_row = self[-1] + return last_row.roll_date + + +def _create_approx_calendar_from_earliest_contract( + earliest_contract_with_roll_data: contractWithRollParametersAndPrices, +) -> pd.DataFrame: + roll_calendar_as_list = _listOfRollCalendarRows() + + # On the roll date we stop holding the current contract, and end up holding the next one + # The roll date is the last day we hold the current contract + dict_of_futures_contract_prices = earliest_contract_with_roll_data.prices + final_contract_date_str = dict_of_futures_contract_prices.last_contract_date_str() + current_contract = earliest_contract_with_roll_data + + while current_contract.date_str < final_contract_date_str: + current_contract.update_expiry_with_offset_from_parameters() + next_contract, new_row = _get_new_row_of_roll_calendar(current_contract) + if new_row is _bad_row: + break + + roll_calendar_as_list.append(new_row) + current_contract = copy(next_contract) + print(current_contract) + + roll_calendar = roll_calendar_as_list.to_pd_df() + + return roll_calendar + + +def _get_new_row_of_roll_calendar( + current_contract: contractWithRollParametersAndPrices, +) -> (contractWithRollParametersAndPrices, _rollCalendarRow): + roll_parameters = current_contract.roll_parameters + final_contract_date_str = current_contract.prices.last_contract_date_str() + + try: + next_contract = current_contract.find_next_held_contract_with_price_data() + except missingData: + # This is a problem UNLESS for the corner case where: + # The current contract isn't the last contract + # But the remaining contracts aren't held contracts + if current_contract.next_held_contract().date_str > final_contract_date_str: + # We are done + return current_contract, _bad_row + else: + raise Exception( + "Can't find good next contract date %s from data when building roll calendar using hold calendar %s" + % ( + current_contract.date_str, + str(roll_parameters.hold_rollcycle), + ) + ) + + try: + carry_contract = current_contract.find_best_carry_contract_with_price_data() + except missingData: + raise Exception( + "Can't find good carry contract %s from data when building roll calendar using hold calendar %s" + % ( + current_contract.date_str, + str(roll_parameters.hold_rollcycle), + ) + ) + + current_roll_date = current_contract.desired_roll_date + new_row = _rollCalendarRow( + current_roll_date, + current_contract.date_str, + next_contract.date_str, + carry_contract.date_str, + ) + + # output initial approx roll calendar to console - gives something to work with if manual adjustment + # is needed + print( + f"{current_roll_date.strftime('%Y-%m-%d %H:%M:00')},{current_contract.date_str},{next_contract.date_str},{carry_contract.date_str}" + ) + # print(new_row) + + return next_contract, new_row + + +localRowData = namedtuple( + "localRowData", ["current_row", "prev_row", "next_row", "first_row_in_data"] +) +_last_row = object() + + +def adjust_to_price_series( + approx_calendar: pd.DataFrame, + dict_of_futures_contract_prices: dictFuturesContractFinalPrices, +) -> pd.DataFrame: + """ + Adjust an approximate roll calendar so that we have matching dates on each expiry for price, carry and next contract + + :param approx_calendar: Approximate roll calendar pd.dataFrame with columns current_contract, next_contract, carry_contract + :param dict_of_futures_contract_prices: dict of futuresContractPrices, keys contract date eg yyyymmdd + + :return: pd.dataFrame with columns current_contract, next_contract + """ + + adjusted_roll_calendar_as_list = _listOfRollCalendarRows() + idx_of_last_row_in_data = len(approx_calendar.index) - 1 + + for row_number in range(len(approx_calendar.index)): + local_row_data = _get_local_data_for_row_number( + approx_calendar, row_number, idx_of_last_row_in_data + ) + if local_row_data is _last_row: + break + + adjusted_row = _adjust_row_of_approx_roll_calendar( + local_row_data, dict_of_futures_contract_prices + ) + + if adjusted_row is _bad_row: + # No suitable roll date was found for this entry. Let's try again but this time + # without requiring prices for carry contracts to be available (Even though carry + # contract is present the price might not necessarily be available on otherwise + # suitable roll dates) + _print_roll_date_carry_warning(local_row_data) + adjusted_row = _adjust_row_of_approx_roll_calendar( + local_row_data, dict_of_futures_contract_prices, omit_carry=True + ) + + if adjusted_row is _bad_row: + _print_roll_date_error(local_row_data) + have_some_data_already = len(adjusted_roll_calendar_as_list) > 0 + if have_some_data_already: + break + else: + ## not at the start yet, let's keep trying for valid data + _print_data_at_start_not_valid_flag(local_row_data) + continue + + adjusted_roll_calendar_as_list.append(adjusted_row) + _print_adjustment_message(local_row_data, adjusted_row) + + if len(adjusted_roll_calendar_as_list) == 0: + raise Exception( + "Error! Empty roll calendar after adjustment! Most likely corrupted roll calendar or maybe using old roll calendar .csv files with new price data?" + ) + + new_calendar = adjusted_roll_calendar_as_list.to_pd_df() + + return new_calendar + + +def _get_local_data_for_row_number( + approx_calendar: pd.DataFrame, row_number: int, idx_of_last_row_in_data: int +) -> localRowData: + last_row_in_data = row_number == idx_of_last_row_in_data + if last_row_in_data: + return _last_row + + first_row_in_data = row_number == 0 + + approx_row = approx_calendar.iloc[row_number, :] + if not first_row_in_data: + prev_approx_row = approx_calendar.iloc[row_number - 1,] + else: + prev_approx_row = _bad_row + + next_approx_row = approx_calendar.iloc[row_number + 1, :] + + local_row_data = localRowData( + approx_row, prev_approx_row, next_approx_row, first_row_in_data + ) + + return local_row_data + + +setOfPrices = namedtuple( + "setOfPrices", + ["current_prices", "next_prices", "curr_carry_prices", "carry_prices"], +) +_no_carry_prices = object() + + +def _adjust_row_of_approx_roll_calendar( + local_row_data: localRowData, + dict_of_futures_contract_prices: dictFuturesContractFinalPrices, + omit_carry: bool = False, +): + roll_date, date_to_avoid = _get_roll_date_and_date_to_avoid(local_row_data) + set_of_prices = _get_set_of_prices( + local_row_data, dict_of_futures_contract_prices, omit_carry + ) + if set_of_prices is _bad_row: + _print_roll_date_error(local_row_data) + return _bad_row + try: + adjusted_roll_date = _find_best_matching_roll_date( + roll_date, + set_of_prices, + avoid_date=date_to_avoid, + ) + except LookupError: + return _bad_row + + adjusted_row = _get_adjusted_row(local_row_data, adjusted_roll_date) + + return adjusted_row + + +def _get_roll_date_and_date_to_avoid(local_row_data: localRowData): + # This is needed to avoid double rolls + approx_row = local_row_data.current_row + prev_approx_row = local_row_data.prev_row + first_row_in_data = local_row_data.first_row_in_data + + roll_date = approx_row.name + if not first_row_in_data: + date_to_avoid = prev_approx_row.name + else: + date_to_avoid = None + + return roll_date, date_to_avoid + + +def _get_set_of_prices( + local_row_data: localRowData, + dict_of_futures_contract_prices: dictFuturesContractFinalPrices, + omit_carry: bool = False, +) -> setOfPrices: + approx_row = local_row_data.current_row + + current_contract = str(approx_row.current_contract) + next_contract = str(approx_row.next_contract) + + try: + current_prices = dict_of_futures_contract_prices[current_contract] + next_prices = dict_of_futures_contract_prices[next_contract] + except KeyError: + return _bad_row + + if omit_carry: + carry_prices = _no_carry_prices + carry_contract = _no_carry_prices + curr_carry_prices = _no_carry_prices + curr_carry_contract = _no_carry_prices + else: + ( + carry_contract, + carry_prices, + curr_carry_contract, + curr_carry_prices, + ) = _get_carry_contract_and_prices( + local_row_data, dict_of_futures_contract_prices + ) + + set_of_prices = setOfPrices( + current_prices, next_prices, curr_carry_prices, carry_prices + ) + + return set_of_prices + + +def _get_carry_contract_and_prices(local_row_data, dict_of_futures_contract_prices): + next_approx_row = local_row_data.next_row + curr_approx_row = local_row_data.current_row + + carry_comes_afterwards = _does_carry_come_after_current_contract(local_row_data) + + if carry_comes_afterwards: + carry_prices = _no_carry_prices + carry_contract = _no_carry_prices + curr_carry_prices = _no_carry_prices + curr_carry_contract = _no_carry_prices + else: + try: + carry_contract = str(next_approx_row.carry_contract) + carry_prices = dict_of_futures_contract_prices[carry_contract] + except KeyError: + carry_prices = _no_carry_prices + try: + curr_carry_contract = str(curr_approx_row.carry_contract) + curr_carry_prices = dict_of_futures_contract_prices[curr_carry_contract] + except KeyError: + curr_carry_prices = _no_carry_prices + + return carry_contract, carry_prices, curr_carry_contract, curr_carry_prices + + +def _does_carry_come_after_current_contract(local_row_data: localRowData) -> bool: + approx_row = local_row_data.current_row + + current_contract = approx_row.current_contract + current_carry_contract = approx_row.carry_contract + + carry_comes_afterwards = current_carry_contract > current_contract + + return carry_comes_afterwards + + +def _print_roll_date_error(local_row_data: localRowData): + approx_row = local_row_data.current_row + next_approx_row = local_row_data.next_row + current_contract = approx_row.current_contract + next_contract = approx_row.next_contract + carry_contract = approx_row.carry_contract + next_carry_contract = next_approx_row.carry_contract + + print( + "Couldn't find matching roll date for contracts %s, %s (even after omitting carry contracts %s and %s)" + % (current_contract, next_contract, carry_contract, next_carry_contract) + ) + print( + "OK if happens at the end or beginning of a roll calendar, otherwise problematic" + ) + + +def _print_roll_date_carry_warning(local_row_data: localRowData): + approx_row = local_row_data.current_row + next_approx_row = local_row_data.next_row + current_contract = approx_row.current_contract + next_contract = approx_row.next_contract + carry_contract = approx_row.carry_contract + next_carry_contract = next_approx_row.carry_contract + + print( + "Warning! Couldn't find matching roll date with concurrent prices for carry contracts (Current: %s Next: %s Carry: %s Next carry: %s)" + % (current_contract, next_contract, carry_contract, next_carry_contract) + ) + print("Now trying to find suitable roll date without requiring carry contracts") + + +def _find_best_matching_roll_date( + roll_date, set_of_prices: setOfPrices, avoid_date=None +): + """ + Find the closest valid roll date for which we have overlapping prices + If avoid_date is passed, get the next date after that + + :param roll_date: datetime.datetime + :param set_of_prices: + :param avoid_date: datetime.datetime or None + + :return: datetime.datetime or + """ + + # Get the list of dates for which a roll is possible + paired_prices = _required_paired_prices(set_of_prices) + valid_dates = _valid_dates_from_paired_prices(paired_prices, avoid_date) + + if len(valid_dates) == 0: + # no matching prices + raise LookupError("No date with a matching price") + + adjusted_date = _find_closest_valid_date_to_approx_roll_date(valid_dates, roll_date) + + return adjusted_date + + +def _required_paired_prices(set_of_prices: setOfPrices) -> pd.DataFrame: + no_carry_exists = set_of_prices.carry_prices is _no_carry_prices + no_curr_carry_exists = set_of_prices.curr_carry_prices is _no_carry_prices + if no_carry_exists or no_curr_carry_exists: + paired_prices = pd.concat( + [set_of_prices.current_prices, set_of_prices.next_prices], axis=1 + ) + else: + paired_prices = pd.concat( + [ + set_of_prices.current_prices, + set_of_prices.next_prices, + set_of_prices.curr_carry_prices, + set_of_prices.carry_prices, + ], + axis=1, + ) + + return paired_prices + + +def _valid_dates_from_paired_prices(paired_prices: pd.DataFrame, avoid_date): + paired_prices_matching = _matching_prices_from_paired_prices(paired_prices) + valid_dates = _valid_dates_from_matching_prices(paired_prices_matching, avoid_date) + + return valid_dates + + +def _matching_prices_from_paired_prices(paired_prices): + paired_prices_check_match = paired_prices.apply( + lambda xlist: not any(np.isnan(xlist)), axis=1 + ) + paired_prices_matching = paired_prices_check_match[paired_prices_check_match] + + return paired_prices_matching + + +def _valid_dates_from_matching_prices(paired_prices_matching, avoid_date): + valid_dates = paired_prices_matching.index + valid_dates.sort_values() + + if avoid_date is not None: + # Remove matching dates before avoid dates + valid_dates = valid_dates[valid_dates > avoid_date] + + return valid_dates + + +def _find_closest_valid_date_to_approx_roll_date(valid_dates, roll_date): + distance_to_roll = valid_dates - roll_date + distance_to_roll_days = [ + abs(distance_item.days) for distance_item in distance_to_roll + ] + closest_date_index = distance_to_roll_days.index(min(distance_to_roll_days)) + adjusted_date = valid_dates[closest_date_index] + + return adjusted_date + + +def _get_adjusted_row( + local_row_data: localRowData, adjusted_roll_date +) -> _rollCalendarRow: + approx_row = local_row_data.current_row + current_carry_contract = approx_row.carry_contract + current_contract = approx_row.current_contract + next_contract = approx_row.next_contract + + adjusted_row = _rollCalendarRow( + adjusted_roll_date, current_contract, next_contract, current_carry_contract + ) + + return adjusted_row + + +def _print_data_at_start_not_valid_flag(local_row_data: localRowData): + approx_row = local_row_data.current_row + print( + "Couldn't get good data for roll date %s but at start so truncating" + % str(approx_row.name) + ) + + +def _print_adjustment_message( + local_row_data: localRowData, adjusted_row: _rollCalendarRow +): + print( + "Changed date from %s to %s for row with contracts %s" + % ( + str(local_row_data.current_row.name), + str(adjusted_row.roll_date), + str(adjusted_row.items()), + ) + ) + + +def _add_carry_calendar( + roll_calendar, roll_parameters_object, dict_of_futures_contract_prices +): + """ + :param roll_calendar: pdDataFrame with current_contract and next_contract + :param roll_parameters_object: rollData + :return: data frame ready to be rollCalendar + """ + + list_of_contract_dates = list(roll_calendar.current_contract.values) + contracts_with_roll_data = [ + contractDateWithRollParameters( + contractDate(str(contract_date)), roll_parameters_object + ) + for contract_date in list_of_contract_dates + ] + + carry_contract_dates = [ + contract.carry_contract().date_str for contract in contracts_with_roll_data + ] + + # Special case if first carry contract missing with a negative offset + first_carry_contract = carry_contract_dates[0] + if first_carry_contract not in dict_of_futures_contract_prices: + # drop the first roll entirely + carry_contract_dates.pop(0) + + # do the same with the calendar or will misalign + first_roll_date = roll_calendar.index[0] + roll_calendar = roll_calendar.drop(labels=first_roll_date) + + roll_calendar["carry_contract"] = carry_contract_dates + + return roll_calendar + + +def back_out_roll_calendar_from_multiple_prices( + multiple_prices: futuresMultiplePrices, +) -> pd.DataFrame: + multiple_prices_unique = multiple_prices[ + ~multiple_prices.index.duplicated(keep="last") + ] + + roll_calendar = _get_roll_calendar_from_unique_prices(multiple_prices_unique) + + roll_calendar = _add_extra_row_to_implied_roll_calendar( + roll_calendar, multiple_prices_unique + ) + + return roll_calendar + + +def _get_roll_calendar_from_unique_prices( + multiple_prices_unique: pd.DataFrame, +) -> pd.DataFrame: + tuple_of_roll_dates = _get_time_indices_from_multiple_prices(multiple_prices_unique) + roll_calendar = _get_roll_calendar_from_roll_dates_and_unique_prices( + multiple_prices_unique, tuple_of_roll_dates + ) + + return roll_calendar + + +def _get_time_indices_from_multiple_prices( + multiple_prices_unique: pd.DataFrame, +) -> tuple: + roll_dates = multiple_prices_unique.index[1:][ + multiple_prices_unique[1:].PRICE_CONTRACT.values + > multiple_prices_unique[:-1].PRICE_CONTRACT.values + ] + days_before = multiple_prices_unique.index[:-1][ + multiple_prices_unique[:-1].PRICE_CONTRACT.values + < multiple_prices_unique[1:].PRICE_CONTRACT.values + ] + + return roll_dates, days_before + + +def _get_roll_calendar_from_roll_dates_and_unique_prices( + multiple_prices_unique: pd.DataFrame, tuple_of_roll_dates: tuple +) -> pd.DataFrame: + roll_dates, days_before = tuple_of_roll_dates + + current_contracts = _extract_contract_from_multiple_prices( + days_before, multiple_prices_unique, "PRICE_CONTRACT" + ) + next_contracts = _extract_contract_from_multiple_prices( + roll_dates, multiple_prices_unique, "PRICE_CONTRACT" + ) + carry_contracts = _extract_contract_from_multiple_prices( + days_before, multiple_prices_unique, "CARRY_CONTRACT" + ) + + roll_calendar = pd.DataFrame( + dict( + current_contract=current_contracts, + next_contract=next_contracts, + carry_contract=carry_contracts, + ), + index=roll_dates, + ) + + return roll_calendar + + +def _extract_contract_from_multiple_prices( + index_of_dates: list, multiple_prices_unique: pd.DataFrame, column_name: str +) -> list: + results = [ + _float_to_contract_str(multiple_prices_unique, date_index, column_name) + for date_index in index_of_dates + ] + return results + + +def _float_to_contract_str(multiple_prices_unique, date_index, column_name): + contract_date = contractDate( + str(int(multiple_prices_unique.loc[date_index][column_name])) + ).date_str + + date_str = contract_date + + return date_str + + +def _add_extra_row_to_implied_roll_calendar( + roll_calendar: pd.DataFrame, multiple_prices_unique: pd.DataFrame +): + final_date = multiple_prices_unique.index[-1] + extra_row = pd.DataFrame( + dict( + current_contract=[ + _float_to_contract_str( + multiple_prices_unique, final_date, "PRICE_CONTRACT" + ) + ], + next_contract=[ + _float_to_contract_str( + multiple_prices_unique, final_date, "FORWARD_CONTRACT" + ) + ], + carry_contract=[ + _float_to_contract_str( + multiple_prices_unique, final_date, "CARRY_CONTRACT" + ) + ], + ), + index=[final_date], + ) + roll_calendar = pd.concat([roll_calendar, extra_row], axis=0) + + return roll_calendar diff --git a/src/quantlib_st/objects/dict_of_futures_per_contract_prices.py b/src/quantlib_st/objects/dict_of_futures_per_contract_prices.py index 5da23f8..0abdd13 100644 --- a/src/quantlib_st/objects/dict_of_futures_per_contract_prices.py +++ b/src/quantlib_st/objects/dict_of_futures_per_contract_prices.py @@ -2,7 +2,7 @@ import numpy as np import pandas as pd -from quantlib_st.core.constants import arg_not_supplied +from quantlib_st.core.constants import arg_not_supplied, named_object from quantlib_st.core.exceptions import missingData from quantlib_st.objects.contract_dates_and_expiries import listOfContractDateStr @@ -48,14 +48,19 @@ def joint_data(self): return joint_data - def matched_prices(self, contracts_to_match=arg_not_supplied) -> pd.DataFrame: + def matched_prices( + self, contracts_to_match: named_object | str | list[str] = arg_not_supplied + ) -> pd.DataFrame: # Return pd.DataFrame where we only have prices in all contracts if contracts_to_match is arg_not_supplied: - contracts_to_match = self.keys() + contracts_to_match = list(self.keys()) + + if isinstance(contracts_to_match, str): + contracts_to_match = [contracts_to_match] joint_data = self.joint_data() - joint_data_to_match = joint_data[contracts_to_match] + joint_data_to_match = joint_data[list(contracts_to_match)] # type: ignore matched_data = joint_data_to_match.dropna() @@ -126,9 +131,9 @@ def daily_volumes(self) -> dictFuturesContractVolumes: def get_last_matched_date_and_prices_for_contract_list( dict_of_prices: dictFuturesContractPrices, - contracts_to_match: list, + contracts_to_match: list[str] | str, list_of_contract_date_str: list, -) -> (datetime.datetime, list): +) -> tuple[datetime.datetime, list]: dict_of_final_prices = dict_of_prices.final_prices() try: diff --git a/src/quantlib_st/objects/dict_of_named_futures_per_contract_prices.py b/src/quantlib_st/objects/dict_of_named_futures_per_contract_prices.py index 203c563..010119d 100644 --- a/src/quantlib_st/objects/dict_of_named_futures_per_contract_prices.py +++ b/src/quantlib_st/objects/dict_of_named_futures_per_contract_prices.py @@ -16,7 +16,9 @@ price_column_names_literal = Literal["PRICE", "CARRY", "FORWARD"] price_column_names = dict(CARRY=carry_name, PRICE=price_name, FORWARD=forward_name) -list_of_price_column_names = list(get_args(price_column_names_literal)) +list_of_price_column_names: list[price_column_names_literal] = list( + get_args(price_column_names_literal) +) list_of_price_column_names.sort() contract_suffix = "_CONTRACT" diff --git a/src/quantlib_st/objects/futures_per_contract_prices.py b/src/quantlib_st/objects/futures_per_contract_prices.py new file mode 100644 index 0000000..3ce6c72 --- /dev/null +++ b/src/quantlib_st/objects/futures_per_contract_prices.py @@ -0,0 +1,235 @@ +import warnings +import pandas as pd +import datetime + +from copy import copy + +from quantlib_st.core.pandas.merge_data_keeping_past_data import SPIKE_IN_DATA +from quantlib_st.core.pandas.frequency import ( + sumup_business_days_over_pd_series_without_double_counting_of_closing_data, +) +from quantlib_st.core.pandas.merge_data_keeping_past_data import merge_newer_data +from quantlib_st.core.pandas.full_merge_with_replacement import ( + full_merge_of_existing_data, +) + +PRICE_DATA_COLUMNS = sorted(["OPEN", "HIGH", "LOW", "FINAL", "VOLUME"]) +FINAL_COLUMN = "FINAL" +VOLUME_COLUMN = "VOLUME" +NOT_VOLUME_COLUMNS = sorted(["OPEN", "HIGH", "LOW", "FINAL"]) + +VERY_BIG_NUMBER = 999999.0 + + +class futuresContractPrices(pd.DataFrame): + """ + simData frame in specific format containing per contract information + """ + + def __init__(self, price_data_as_df: pd.DataFrame): + """ + + :param data: pd.DataFrame or something that could be passed to it + """ + + _validate_price_data(price_data_as_df) + price_data_as_df.index.name = "index" # for arctic compatibility + super().__init__(price_data_as_df) + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning) + self._as_df = price_data_as_df + + def __copy__(self): + return futuresContractPrices(copy(self._as_df)) + + @classmethod + def create_empty(futuresContractPrices): + """ + Our graceful fail is to return an empty, but valid, dataframe + """ + + data = pd.DataFrame(columns=PRICE_DATA_COLUMNS) + + futures_contract_prices = futuresContractPrices(data) + + return futures_contract_prices + + @classmethod + def create_from_final_prices_only(cls, price_data_as_series: pd.Series): + price_data_as_series = pd.DataFrame( + price_data_as_series, columns=[FINAL_COLUMN] + ) + price_data_as_series = price_data_as_series.reindex(columns=PRICE_DATA_COLUMNS) + + futures_contract_prices = cls(price_data_as_series) + + return futures_contract_prices + + def return_final_prices(self): + data = self[FINAL_COLUMN] + + return futuresContractFinalPrices(data) + + def _raw_volumes(self) -> pd.Series: + data = self[VOLUME_COLUMN] + + return data + + def inverse(self): + new_version = copy(self) + for colname in NOT_VOLUME_COLUMNS: + new_version[colname] = 1 / self[colname] + + return futuresContractPrices(new_version) + + def multiply_prices(self, multiplier: float): + new_version = copy(self) + for colname in NOT_VOLUME_COLUMNS: + new_version[colname] = multiplier * self[colname] + + return futuresContractPrices(new_version) + + def add_offset_to_prices(self, offset: float): + new_version = copy(self) + for colname in NOT_VOLUME_COLUMNS: + new_version[colname] = offset + self[colname] + + return futuresContractPrices(new_version) + + def daily_volumes(self) -> pd.Series: + volumes = self._raw_volumes() + + # stop double counting + daily_volumes = ( + sumup_business_days_over_pd_series_without_double_counting_of_closing_data( + volumes + ) + ) + + return daily_volumes + + def merge_with_other_prices( + self, + new_futures_per_contract_prices, + only_add_rows=True, + check_for_spike=True, + keep_older: bool = True, + ): + """ + Merges self with new data. + If only_add_rows is True, + Otherwise: Any Nan in the existing data will be replaced (be careful!) + + :param new_futures_per_contract_prices: another futures per contract prices object + :param keep_older: bool. Keep older data if not NaN (default). False : overwrite older data with non-NaN values. Applicable only to full merge (only_add_rows=False) + :param check_for_spike Checks for data spikes. + :return: merged futures_per_contract object + """ + if only_add_rows: + return self.add_rows_to_existing_data( + new_futures_per_contract_prices, check_for_spike=check_for_spike + ) + else: + return self._full_merge_of_existing_data( + new_futures_per_contract_prices, + check_for_spike=check_for_spike, + keep_older=keep_older, + ) + + def _full_merge_of_existing_data( + self, + new_futures_per_contract_prices, + check_for_spike=False, + keep_older: bool = True, + ): + """ + Merges self with new data. + Any Nan in the existing data will be replaced (be careful!) + + :param new_futures_per_contract_prices: the new data + :param check_for_spike Checks for data spikes. + :param keep_older: bool. Keep older data (default). + :return: updated data, doesn't update self + """ + + merged_data = full_merge_of_existing_data( + self, + new_futures_per_contract_prices, + keep_older=keep_older, + check_for_spike=check_for_spike, + column_to_check_for_spike=FINAL_COLUMN, + ) + + if merged_data is SPIKE_IN_DATA: + return SPIKE_IN_DATA + + return futuresContractPrices(merged_data) + + def remove_zero_volumes(self): + drop_it = self[VOLUME_COLUMN] == 0 + new_data = self[~drop_it] + return futuresContractPrices(new_data) + + def remove_zero_prices(self): + drop_it = self[FINAL_COLUMN] == 0.0 + new_data = self[~drop_it] + return futuresContractPrices(new_data) + + def remove_negative_prices(self): + drop_it = self[FINAL_COLUMN] < 0.0 + new_data = self[~drop_it] + return futuresContractPrices(new_data) + + def remove_future_data(self): + new_data = futuresContractPrices(self[self.index < datetime.datetime.now()]) + + return new_data + + def add_rows_to_existing_data( + self, + new_futures_per_contract_prices, + check_for_spike=True, + max_price_spike: float = VERY_BIG_NUMBER, + ): + """ + Merges self with new data. + Only newer data will be added + + :param new_futures_per_contract_prices: another futures per contract prices object + + :return: merged futures_per_contract object + """ + + merged_futures_prices = merge_newer_data( + pd.DataFrame(self), + new_futures_per_contract_prices, + check_for_spike=check_for_spike, + max_spike=max_price_spike, + column_to_check_for_spike=FINAL_COLUMN, + ) + + if merged_futures_prices is SPIKE_IN_DATA: + return SPIKE_IN_DATA + + merged_futures_prices = futuresContractPrices(merged_futures_prices) + + return merged_futures_prices + + +class futuresContractFinalPrices(pd.Series): + """ + Just the final prices from a futures contract + """ + + def __init__(self, data): + super().__init__(data) + + +def _validate_price_data(data: pd.DataFrame): + data_present = sorted(data.columns) + + try: + assert data_present == PRICE_DATA_COLUMNS + except AssertionError: + raise Exception("futuresContractPrices has to conform to pattern") diff --git a/src/quantlib_st/objects/multiple_prices.py b/src/quantlib_st/objects/multiple_prices.py index 61bfce9..841eba6 100644 --- a/src/quantlib_st/objects/multiple_prices.py +++ b/src/quantlib_st/objects/multiple_prices.py @@ -78,10 +78,10 @@ def __init__(self, data): @classmethod ## NOT TYPE CHECKING OF ROLL_CALENDAR AS WOULD CAUSE CIRCULAR IMPORT def create_from_raw_data( - futuresMultiplePrices, + cls, roll_calendar, dict_of_futures_contract_closing_prices: dictFuturesContractFinalPrices, - ): + ) -> "futuresMultiplePrices": """ :param roll_calendar: rollCalendar @@ -94,20 +94,20 @@ def create_from_raw_data( roll_calendar, dict_of_futures_contract_closing_prices ) - multiple_prices = futuresMultiplePrices(all_price_data_stack) + multiple_prices = cls(all_price_data_stack) multiple_prices._is_empty = False return multiple_prices @classmethod - def create_empty(futuresMultiplePrices): + def create_empty(cls) -> "futuresMultiplePrices": """ Our graceful fail is to return an empty, but valid, dataframe """ data = pd.DataFrame(columns=multiple_data_columns) - multiple_prices = futuresMultiplePrices(data) + multiple_prices = cls(data) return multiple_prices @@ -116,7 +116,7 @@ def inverse(self): for colname in list_of_price_column_names: new_version[colname] = 1 / self[colname] - return futuresMultiplePrices(new_version) + return type(self)(new_version) def add_offset_to_prices(self, offset: float): new_version = copy(self) @@ -161,9 +161,9 @@ def as_dict(self) -> dictFuturesNamedContractFinalPricesWithContractID: @classmethod def from_merged_dict( - futuresMultiplePrices, + cls, prices_dict: dictFuturesNamedContractFinalPricesWithContractID, - ): + ) -> "futuresMultiplePrices": """ Re-create from dict, eg results of _as_dict @@ -191,7 +191,7 @@ def from_merged_dict( multiple_prices_data_frame[list_of_contract_column_names].ffill() ) - multiple_prices_object = futuresMultiplePrices(multiple_prices_data_frame) + multiple_prices_object = cls(multiple_prices_data_frame) return multiple_prices_object diff --git a/src/quantlib_st/objects/roll_calendars.py b/src/quantlib_st/objects/roll_calendars.py new file mode 100644 index 0000000..47f537f --- /dev/null +++ b/src/quantlib_st/objects/roll_calendars.py @@ -0,0 +1,173 @@ +import pandas as pd +import numpy as np + +from quantlib_st.init.futures.build_roll_calendars import ( + generate_approximate_calendar, + adjust_to_price_series, + back_out_roll_calendar_from_multiple_prices, +) +from quantlib_st.objects.dict_of_futures_per_contract_prices import ( + dictFuturesContractFinalPrices, +) +from quantlib_st.objects.multiple_prices import futuresMultiplePrices + +from quantlib_st.objects.rolls import rollParameters + + +class rollCalendar(pd.DataFrame): + """ + A roll calendar is a dataframe telling us when we have rolled futures contracts in the past (or would have in a backtest) + + It has a datetime index, and two columns; current_contract and next_contract + + Normally a roll calendar would be created using the following process: (and this is what __init__ does) + - start with a list of futures contracts and some rollParameters + - using a rollParameters object we work out roughly what the rolls should be (in an ideal world with available prices all the time) + - then using a list of futures contract price data we shift the rolls around so that rolls are possible on a given date + + Another way of getting a roll calendar is to back it out from an existing 'carry data' (eg as we have in legacy csv) + + Sometimes you need to manually hack roll calendars, so it's also useful to have a csv convenience method for read/write + + When combined with a list of futures contract price data a roll calendar can be used to create a back adjusted price series + This can then be stored. + (We don't create these 'on line' as it's a bit slow. We can add additional rows to a back adjusted price series just + from the current price. Then the re-adjustment can happen again on each roll. Could use Arctic vintage method here?) + + """ + + @classmethod + def create_from_prices( + cls, + dict_of_futures_contract_prices: dictFuturesContractFinalPrices, + roll_parameters_object: rollParameters, + ): + """ + + :param dict_of_futures_contract_prices: dict, keys are contract date ids 'yyyymmdd' + :param roll_parameters_object: roll parameters specific to this instrument + """ + + approx_calendar = generate_approximate_calendar( + roll_parameters_object, dict_of_futures_contract_prices + ) + + adjusted_calendar = adjust_to_price_series( + approx_calendar, dict_of_futures_contract_prices + ) + + roll_calendar = cls(adjusted_calendar) + + return roll_calendar + + @classmethod + def back_out_from_multiple_prices(cls, multiple_prices: futuresMultiplePrices): + """ + + :param multiple_prices: output from futuresDataForSim.FuturesData.get_current_and_forward_price_data(instrument_code) + columns: PRICE, FORWARD, FORWARD_CONTRACT, PRICE_CONTRACT + + :return: rollCalendar + """ + roll_calendar_as_pd = back_out_roll_calendar_from_multiple_prices( + multiple_prices + ) + roll_calendar_object = cls(roll_calendar_as_pd) + + return roll_calendar_object + + def check_if_date_index_monotonic(self) -> bool: + if not (self.index.is_monotonic_increasing and self.index.is_unique): + print( + "WARNING: Date index not monotonically increasing in following indices:" + ) + + not_monotonic = self.index[1:][self.index[1:] <= self.index[:-1]] + print(not_monotonic) + + return False + else: + return True + + def check_dates_are_valid_for_prices( + self, dict_of_futures_contract_prices: dictFuturesContractFinalPrices + ) -> bool: + """ + Adjust an approximate roll calendar so that we have matching dates on each expiry + + :param dict_of_futures_contract_prices: dict of futuresContractPrices, keys contract date eg yyyymmdd + + :return: bool, True if no problems + """ + + checks_okay = True + for row_number in range(len(self.index)): + calendar_row = self.iloc[row_number, :] + + checks_okay_this_row = _check_row_of_row_calendar( + calendar_row, dict_of_futures_contract_prices + ) + + if not checks_okay_this_row: + # single failure is a total failure + checks_okay = False + + return checks_okay + + +def _check_row_of_row_calendar( + calendar_row: pd.Series, + dict_of_futures_contract_prices: dictFuturesContractFinalPrices, +) -> bool: + current_contract = str(calendar_row.current_contract) + next_contract = str(calendar_row.next_contract) + carry_contract = str(calendar_row.carry_contract) + roll_date = calendar_row.name + + try: + current_prices = dict_of_futures_contract_prices[current_contract] + except KeyError: + print( + "On roll date %s contract %s is missing from futures prices" + % (roll_date, current_contract) + ) + return False + try: + next_prices = dict_of_futures_contract_prices[next_contract] + except KeyError: + print( + "On roll date %s contract %s is missing from futures prices" + % (roll_date, next_contract) + ) + return False + + try: + carry_prices = dict_of_futures_contract_prices[carry_contract] + except KeyError: + print( + "On roll date %s contract %s is missing from futures prices" + % (roll_date, carry_contract) + ) + return False + + try: + current_price = current_prices.loc[roll_date] + except KeyError: + print("Roll date %s missing from prices for %s" % (roll_date, current_contract)) + return False + + try: + next_price = next_prices.loc[roll_date] + except KeyError: + print("Roll date %s missing from prices for %s" % (roll_date, next_contract)) + return False + + if np.isnan(current_price): + print("NAN for price on %s for %s " % (roll_date, current_contract)) + return False + + if np.isnan(next_price): + print("NAN for price on %s for %s " % (roll_date, current_contract)) + return False + + return True diff --git a/src/quantlib_st/objects/roll_parameters_with_price_data.py b/src/quantlib_st/objects/roll_parameters_with_price_data.py new file mode 100644 index 0000000..2d54bb2 --- /dev/null +++ b/src/quantlib_st/objects/roll_parameters_with_price_data.py @@ -0,0 +1,305 @@ +import datetime + +from quantlib_st.core.exceptions import missingData +from quantlib_st.objects.rolls import contractDateWithRollParameters, rollParameters +from quantlib_st.objects.contracts import contractDate +from quantlib_st.objects.dict_of_futures_per_contract_prices import ( + dictFuturesContractFinalPrices, +) +from quantlib_st.objects.contract_dates_and_expiries import listOfContractDateStr + +HELD = "held" +PRICED = "priced" + + +class contractWithRollParametersAndPrices(object): + """ + Including prices in our contract means we can navigate more accurately through roll cycles + """ + + def __init__( + self, + contract_with_roll_parameters: contractDateWithRollParameters, + dict_of_final_price_data: dictFuturesContractFinalPrices, + ): + """ + + :param contract_with_roll_parameters: contractWithRollParameters + :param dict_of_final_price_data: object of type dictFuturesContractFinalPrices + """ + + self._contract = contract_with_roll_parameters + self._prices = dict_of_final_price_data + + @property + def contract(self): + return self._contract + + @property + def prices(self): + return self._prices + + @property + def roll_parameters(self): + return self.contract.roll_parameters + + @property + def date_str(self) -> str: + return self.contract.date_str + + @property + def desired_roll_date(self) -> datetime.datetime: + return self.contract.desired_roll_date + + def update_expiry_with_offset_from_parameters(self): + expiry_offset = self.roll_parameters.approx_expiry_offset + self.contract.contract_date.update_expiry_date_with_new_offset(expiry_offset) + + def next_held_contract(self): + next_held_contract_with_roll_parameters = self.contract.next_held_contract() + return contractWithRollParametersAndPrices( + next_held_contract_with_roll_parameters, self.prices + ) + + def next_priced_contract(self): + next_priced_contract_with_roll_parameters = self.contract.next_priced_contract() + return contractWithRollParametersAndPrices( + next_priced_contract_with_roll_parameters, self.prices + ) + + def previous_priced_contract(self): + previous_priced_contract_with_roll_parameters = ( + self.contract.previous_priced_contract() + ) + return contractWithRollParametersAndPrices( + previous_priced_contract_with_roll_parameters, self.prices + ) + + def previous_held_contract(self): + previous_held_contract_with_roll_parameters = ( + self.contract.previous_held_contract() + ) + return contractWithRollParametersAndPrices( + previous_held_contract_with_roll_parameters, self.prices + ) + + def find_next_held_contract_with_price_data(self): + """ + Finds the first contract in list_of_contract_dates after current_contract, within the held roll cycle + defined by roll parameters + + :return: a contract object with roll data, or None if we can't find one + """ + next_contract = self._find_next_contract_with_price_data(HELD) + + return next_contract + + def find_next_priced_contract_with_price_data(self): + """ + Finds the first contract in list_of_contract_dates after current_contract, within the priced roll cycle + defined by roll parameters + + :return: a contract object with roll data, or None if we can't find one + """ + + next_contract = self._find_next_contract_with_price_data(PRICED) + + return next_contract + + def _find_next_contract_with_price_data(self, contract_type: str): + """ + Finds the first contract in list_of_contract_dates after current_contract, within the priced roll cycle + defined by roll parameters + + :return: a contract object with roll data, or None if we can't find one + """ + assert contract_type in [HELD, PRICED] + contract_attribute_str = "next_%s_contract" % contract_type + + try_contract = getattr(self, contract_attribute_str)() + list_of_contract_dates = self.prices.sorted_contract_date_str() + final_contract_date = list_of_contract_dates[-1] + + while try_contract.date_str <= final_contract_date: + if try_contract.date_str in list_of_contract_dates: + return try_contract + else: + if contract_type == HELD: + roll_cycle = self.roll_parameters.hold_rollcycle + else: + roll_cycle = self.roll_parameters.priced_rollcycle + print( + "Warning! After", + self.date_str, + "the next expected contract", + try_contract.date_str, + "in the", + contract_type, + "roll cycle (", + roll_cycle, + ") not available! (OK if this is at the end of the calendar)", + ) + try_contract = getattr(try_contract, contract_attribute_str)() + + # Nothing found + raise missingData + + def find_previous_priced_contract_with_price_data(self): + """ + Finds the closest contract in list_of_contract_dates before current_contract, within the priced roll cycle + defined by roll parameters + + :return: a contract object with roll data, or None if we can't find one + """ + previous_contract = self._find_previous_contract_with_price_data(PRICED) + + return previous_contract + + def find_previous_held_contract_with_price_data(self): + """ + Finds the closest contract in list_of_contract_dates before current_contract, within the held roll cycle + defined by roll parameters + + :return: a contract object with roll data, or None if we can't find one + """ + previous_contract = self._find_previous_contract_with_price_data(HELD) + + return previous_contract + + def _find_previous_contract_with_price_data(self, contract_type: str): + """ + Finds the closest contract in list_of_contract_dates before current_contract, within the held roll cycle + defined by roll parameters + + :return: a contract object with roll data, or None if we can't find one + """ + assert contract_type in [HELD, PRICED] + contract_attribute_str = "previous_%s_contract" % contract_type + + try_contract = getattr(self, contract_attribute_str)() + list_of_contract_dates = self.prices.sorted_contract_date_str() + first_contract_date = list_of_contract_dates[0] + + while try_contract.date_str >= first_contract_date: + if try_contract.date_str in list_of_contract_dates: + return try_contract + else: + if contract_type == HELD: + roll_cycle = self.roll_parameters.hold_rollcycle + else: + roll_cycle = self.roll_parameters.priced_rollcycle + print( + "Warning! Before", + self.date_str, + "the previous expected contract", + try_contract.date_str, + "in the", + contract_type, + "roll cycle (", + roll_cycle, + ") not available! (OK if this is at the end of the calendar)", + ) + try_contract = getattr(try_contract, contract_attribute_str)() + + # Nothing found + raise missingData + + def find_best_carry_contract_with_price_data(self): + """ + Finds the best carry contract in list_of_contract_dates after current_contract, within the roll cycle + defined by roll parameters + + This will either be the next valid contract, or the first valid preceeding contract in the price cycle + + :return: a contract object with roll data, or None if we can't find one + """ + carry_offset = self.contract.roll_parameters.carry_offset + + if carry_offset == 1.0: + best_carry_contract = self.find_next_priced_contract_with_price_data() + elif carry_offset == -1.0: + best_carry_contract = self.find_previous_priced_contract_with_price_data() + else: + raise Exception("Carry offset should be 1 or -1!") + + return best_carry_contract + + +def find_earliest_held_contract_with_price_data( + roll_parameters_object: rollParameters, price_dict: dictFuturesContractFinalPrices +) -> contractWithRollParametersAndPrices: + """ + Find the earliest contract we can hold in a given list of contract dates + To hold the contract, it needs to be in the held roll cycle and the list_of_contract_dates + And it's carry contract needs to be in the priced roll cycle and the list_of_contract_dates + + :return: contract with roll parameters, or None + """ + list_of_contract_dates = price_dict.sorted_contract_date_str() + + earliest_contract = _find_earliest_held_contract_with_data( + list_of_contract_dates, roll_parameters_object, price_dict + ) + + return earliest_contract + + +def _find_earliest_held_contract_with_data( + list_of_contract_dates: listOfContractDateStr, + roll_parameters_object: rollParameters, + price_dict: dictFuturesContractFinalPrices, +) -> contractWithRollParametersAndPrices: + try_contract = _initial_contract_to_try_with( + list_of_contract_dates, roll_parameters_object, price_dict + ) + final_contract_date = list_of_contract_dates[-1] + + while try_contract.date_str <= final_contract_date: + is_contract_ok = _check_valid_contract(try_contract, list_of_contract_dates) + # Okay this works + if is_contract_ok: + return try_contract + + # okay it's not suitable + # Let's try another one + try_contract = try_contract.find_next_held_contract_with_price_data() + + # Nothing found + raise missingData + + +def _initial_contract_to_try_with( + list_of_contract_dates: list, + roll_parameters_object: rollParameters, + price_dict: dictFuturesContractFinalPrices, +) -> contractWithRollParametersAndPrices: + plausible_earliest_contract_date = list_of_contract_dates[0] + plausible_earliest_contract = contractDateWithRollParameters( + contractDate( + plausible_earliest_contract_date, + approx_expiry_offset=roll_parameters_object.approx_expiry_offset, + ), + roll_parameters_object, + ) + + try_contract = contractWithRollParametersAndPrices( + plausible_earliest_contract, price_dict + ) + + return try_contract + + +def _check_valid_contract( + try_contract: contractWithRollParametersAndPrices, + list_of_contract_dates: listOfContractDateStr, +) -> bool: + if try_contract.date_str in list_of_contract_dates: + # possible candidate, let's check carry + try: + try_carry_contract = try_contract.find_best_carry_contract_with_price_data() + except missingData: + ## No good + return False + + ## All good + return True diff --git a/src/quantlib_st/objects/rolls.py b/src/quantlib_st/objects/rolls.py index 067a5b7..58a2efb 100644 --- a/src/quantlib_st/objects/rolls.py +++ b/src/quantlib_st/objects/rolls.py @@ -58,7 +58,7 @@ def iterate_contract_date( def _previous_year_month_given_tuple( self, year_value: int, month_str: str - ) -> (int, str): + ) -> tuple[int, str]: """ Returns a tuple (year, month: str) @@ -75,7 +75,7 @@ def _previous_year_month_given_tuple( def _next_year_month_given_tuple( self, year_value: int, month_str: str - ) -> (int, str): + ) -> tuple[int, str]: """ Returns a tuple (year, month: str) @@ -261,8 +261,8 @@ def global_rollcycle(self): return self._global_rollcycle @classmethod - def create_from_dict(rollData, roll_data_dict: dict): - futures_instrument_roll_data = rollData(**roll_data_dict) + def create_from_dict(cls, roll_data_dict: dict): + futures_instrument_roll_data = cls(**roll_data_dict) return futures_instrument_roll_data diff --git a/src/quantlib_st/sysdata/base_data.py b/src/quantlib_st/sysdata/base_data.py index b418249..963752c 100644 --- a/src/quantlib_st/sysdata/base_data.py +++ b/src/quantlib_st/sysdata/base_data.py @@ -1,3 +1,5 @@ +import pandas as pd + from quantlib_st.logging.logger import * @@ -33,14 +35,14 @@ def __init__(self, log=get_logger("baseData")): self._log = log - def __repr__(self): + def __repr__(self) -> str: return "baseData object" @property def log(self): return self._log - def __getitem__(self, keyname): + def __getitem__(self, keyname) -> pd.DataFrame: """ convenience method to get the price, make it look like a dict diff --git a/src/quantlib_st/sysdata/csv/csv_futures_contract_prices.py b/src/quantlib_st/sysdata/csv/csv_futures_contract_prices.py new file mode 100644 index 0000000..5d6fb1a --- /dev/null +++ b/src/quantlib_st/sysdata/csv/csv_futures_contract_prices.py @@ -0,0 +1,300 @@ +from dataclasses import dataclass + +from quantlib_st.sysdata.futures.futures_per_contract_prices import ( + futuresContractPriceData, +) +from quantlib_st.objects.futures_per_contract_prices import futuresContractPrices +from quantlib_st.objects.contracts import futuresContract, listOfFuturesContracts +from quantlib_st.logging.logger import * +from quantlib_st.core.fileutils import ( + resolve_path_and_filename_for_package, + files_with_extension_in_pathname, +) +from quantlib_st.core.constants import arg_not_supplied, named_object +from quantlib_st.core.dateutils import MIXED_FREQ, Frequency +from quantlib_st.core.pandas.pdutils import pd_readcsv, DEFAULT_DATE_FORMAT_FOR_CSV + + +@dataclass +class ConfigCsvFuturesPrices: + input_date_index_name: str = "DATETIME" + input_date_format: str = DEFAULT_DATE_FORMAT_FOR_CSV + input_column_mapping: dict | named_object = arg_not_supplied + input_skiprows: int = 0 + input_skipfooter: int = 0 + apply_multiplier: float = 1.0 + apply_inverse: bool = False + + +class csvFuturesContractPriceData(futuresContractPriceData): + """ + Class to read / write individual futures contract price data to and from csv files + # no default datapath supplied as this is not normally used + """ + + def __init__( + self, + datapath=arg_not_supplied, + log=get_logger("csvFuturesContractPriceData"), + config: ConfigCsvFuturesPrices | named_object = arg_not_supplied, + ): + super().__init__(log=log) + if datapath is arg_not_supplied: + raise Exception("Need to pass datapath") + self._datapath = datapath + if config is arg_not_supplied: + config = ConfigCsvFuturesPrices() + + self._config = config + + def __repr__(self): + return "csvFuturesContractPricesData accessing %s" % self._datapath + + def _get_merged_prices_for_contract_object_no_checking( + self, contract_object: futuresContract + ) -> futuresContractPrices: + """ + Read back the prices for a given contract object + + :param futures_contract_object: futuresContract + :return: data + """ + + return self._get_prices_at_frequency_for_contract_object_no_checking( + futures_contract_object=contract_object, frequency=MIXED_FREQ + ) + + def _get_prices_at_frequency_for_contract_object_no_checking( + self, futures_contract_object: futuresContract, frequency: Frequency + ) -> futuresContractPrices: + keyname = self._keyname_given_contract_object_and_freq( + futures_contract_object, frequency=frequency + ) + filename = self._filename_given_key_name(keyname) + config = self.config + + assert config is ConfigCsvFuturesPrices + + date_format = config.input_date_format + date_time_column = config.input_date_index_name + input_column_mapping = config.input_column_mapping + skiprows = config.input_skiprows + skipfooter = config.input_skipfooter + multiplier = config.apply_multiplier + inverse = config.apply_inverse + + try: + instrpricedata = pd_readcsv( + filename, + date_index_name=date_time_column, + date_format=date_format, + input_column_mapping=input_column_mapping, + skiprows=skiprows, + skipfooter=skipfooter, + ) + except OSError: + self.log.warning( + "Can't find adjusted price file %s" % filename, + **futures_contract_object.log_attributes(), # type: ignore + method="temp", + ) + return futuresContractPrices.create_empty() + + instrpricedata = instrpricedata.groupby(level=0).last() + for col_name in ["OPEN", "HIGH", "LOW", "FINAL"]: + column_series = instrpricedata[col_name] + if inverse: + column_series = 1 / column_series + column_series *= multiplier + + instrpricedata = futuresContractPrices(instrpricedata) + + return instrpricedata + + def _write_merged_prices_for_contract_object_no_checking( + self, + futures_contract_object: futuresContract, + futures_price_data: futuresContractPrices | named_object, + ): + """ + Write prices + CHECK prices are overridden on second write + + :param futures_contract_object: futuresContract + :param futures_price_data: futuresContractPriceData + :return: None + """ + self._write_prices_at_frequency_for_contract_object_no_checking( + futures_contract_object=futures_contract_object, + futures_price_data=futures_price_data, + frequency=MIXED_FREQ, + ) + + def _write_prices_at_frequency_for_contract_object_no_checking( + self, + futures_contract_object: futuresContract, + futures_price_data: futuresContractPrices | named_object, + frequency: Frequency, + ): + keyname = self._keyname_given_contract_object_and_freq( + futures_contract_object, frequency=frequency + ) + filename = self._filename_given_key_name(keyname) + assert isinstance(self.config, ConfigCsvFuturesPrices) + assert isinstance(futures_price_data, futuresContractPrices) + futures_price_data.to_csv( + filename, index_label=self.config.input_date_index_name + ) + + def _delete_merged_prices_for_contract_object_with_no_checks_be_careful( + self, futures_contract_object: futuresContract + ): + raise NotImplementedError( + "You can't delete futures prices stored as a csv - Add to overwrite existing data, or delete file manually" + ) + + def _delete_prices_at_frequency_for_contract_object_with_no_checks_be_careful( + self, futures_contract_object: futuresContract, frequency: Frequency + ): + raise NotImplementedError( + "You can't delete futures prices stored as a csv - Add to overwrite existing data, or delete file manually" + ) + + def has_merged_price_data_for_contract( + self, contract_object: futuresContract + ) -> bool: + return self.has_price_data_for_contract_at_frequency( + contract_object, frequency=MIXED_FREQ + ) + + def has_price_data_for_contract_at_frequency( + self, contract_object: futuresContract, frequency: Frequency + ) -> bool: + return ( + self._keyname_given_contract_object_and_freq( + contract_object, frequency=frequency + ) + in self._all_keynames_in_library() + ) + + def get_contracts_with_merged_price_data(self) -> listOfFuturesContracts: + """ + + :return: list of contracts + """ + + return self.get_contracts_with_price_data_for_frequency(frequency=MIXED_FREQ) + + def get_contracts_with_price_data_for_frequency( + self, frequency: Frequency + ) -> listOfFuturesContracts: + list_of_contract_and_freq_tuples = ( + self._get_contract_freq_tuples_with_price_data() + ) + + list_of_contracts = [ + futuresContract(contract_freq_tuple[1], contract_freq_tuple[2]) + for contract_freq_tuple in list_of_contract_and_freq_tuples + if contract_freq_tuple[0] == frequency + ] + + list_of_contracts = listOfFuturesContracts(list_of_contracts) + + return list_of_contracts + + def _get_contract_freq_tuples_with_price_data(self) -> list: + """ + + :return: list of futures contracts as tuples + """ + + all_keynames = self._all_keynames_in_library() + list_of_contract_and_freq_tuples = [ + self._contract_tuple_and_freq_given_keyname(keyname) + for keyname in all_keynames + ] + + return list_of_contract_and_freq_tuples + + def _keyname_given_contract_object_and_freq( + self, futures_contract_object: futuresContract, frequency: Frequency + ) -> str: + """ + We could do this using the .ident() method of instrument_object, but this way we keep control inside this class + + :param futures_contract_object: futuresContract + :return: str + """ + if frequency is MIXED_FREQ: + frequency_str = "" + else: + frequency_str = frequency.name + "_" + + instrument_str = str(futures_contract_object.instrument) + date_str = str(futures_contract_object.date_str) + + return "%s%s_%s" % (frequency_str, instrument_str, date_str) + + def _contract_tuple_and_freq_given_keyname(self, keyname: str) -> tuple: + """ + Extract the two parts of a keyname + + We keep control of how we represent stuff inside the class + + :param keyname: str + :return: tuple instrument_code, contract_date + """ + if keyname.startswith("Day") or keyname.startswith("Hour"): + ## has frequency + index = keyname.find("_") + frequency = Frequency[keyname[:index]] + residual_keyname = keyname[index + 1 :] + else: + ## no frequency, mixed data + frequency = MIXED_FREQ + residual_keyname = keyname + + keyname_as_list = residual_keyname.split("_") + + if len(keyname_as_list) == 4: + keyname_as_list = [ + "%s_%s_%s" + % (keyname_as_list[0], keyname_as_list[1], keyname_as_list[2]), + keyname_as_list[3], + ] + + # It's possible to have GAS_US_20090700.csv, so we only take the second + if len(keyname_as_list) == 3: + keyname_as_list = [ + "%s_%s" % (keyname_as_list[0], keyname_as_list[1]), + keyname_as_list[2], + ] + + try: + assert len(keyname_as_list) == 2 + except BaseException: + self.log.error( + "Keyname (filename) %s in wrong format should be instrument_contractid" + % keyname + ) + instrument_code, contract_date = tuple(keyname_as_list) + + return frequency, instrument_code, contract_date + + def _filename_given_key_name(self, keyname: str): + assert isinstance(self._datapath, str) + return resolve_path_and_filename_for_package( + self._datapath, "%s.csv" % (keyname) + ) + + def _all_keynames_in_library(self) -> list[str]: + assert isinstance(self._datapath, str) + return files_with_extension_in_pathname(self._datapath, ".csv") + + @property + def config(self): + return self._config + + @property + def datapath(self): + return self._datapath diff --git a/src/quantlib_st/sysdata/futures/README.md b/src/quantlib_st/sysdata/futures/README.md new file mode 100644 index 0000000..105e55d --- /dev/null +++ b/src/quantlib_st/sysdata/futures/README.md @@ -0,0 +1,99 @@ +# Futures Contract Price Data Implementation Guide + +This directory contains the base class and implementations for handling per-contract futures price data. + +To implement a new data source (e.g., an API-based provider like Bloomberg, Reuters, or an internal database), you should inherit from `futuresContractPriceData` in `futures_per_contract_prices.py`. + +## Core Implementation Requirements + +When creating a new subclass, you must implement the following "no checking" internal methods which the base class uses to satisfy public API requests. + +### 1. Data Retrieval (Mandatory) + +#### `_get_merged_prices_for_contract_object_no_checking(self, contract_object: futuresContract) -> futuresContractPrices` + +Fetches all available price data for a specific contract. + +- **Input**: A `futuresContract` object (contains `instrument_code` and `date_str`). +- **Output**: A `futuresContractPrices` object. + +#### `_get_prices_at_frequency_for_contract_object_no_checking(self, contract_object: futuresContract, frequency: Frequency) -> futuresContractPrices` + +Fetches price data at a specific granularity (Daily, Hourly, etc.). + +- **Input**: `futuresContract` object and a `Frequency` enum. +- **Output**: A `futuresContractPrices` object. + +### 2. Discovery Methods (Mandatory) + +#### `get_contracts_with_merged_price_data(self) -> listOfFuturesContracts` + +Returns a complete list of all contracts available in the data source. + +#### `get_contracts_with_price_data_for_frequency(self, frequency: Frequency) -> listOfFuturesContracts` + +Returns a list of all contracts available for a specific frequency. + +## The `listOfFuturesContracts` Object + +Methods that return contract lists must return a `listOfFuturesContracts` object (from `quantlib_st.objects.contracts`). This is a specialized list of `futuresContract` objects that provides utility methods for filtering. + +### Creating the list + +To implement these methods, you typically create a standard Python list of `futuresContract` objects and then wrap it: + +```python +from quantlib_st.objects.contracts import futuresContract, listOfFuturesContracts + +def get_contracts_with_merged_price_data(self) -> listOfFuturesContracts: + # 1. Logic to get your ticker list (e.g. from an API or database) + all_tickers = [('CRUDE_OIL', '20240600'), ('CRUDE_OIL', '20240900')] + + # 2. Convert to futuresContract objects + contract_list = [ + futuresContract(instr, date_id) + for instr, date_id in all_tickers + ] + + # 3. Return as the specialized collection + return listOfFuturesContracts(contract_list) +``` + +### Why use `listOfFuturesContracts`? + +The system relies on this object's helper methods, such as: + +- `.unique_list_of_instrument_codes()`: To see which instruments have data. +- `.contracts_in_list_for_instrument_code(code)`: To filter contracts for a single instrument. +- `.list_of_dates()`: To get a list of YYYYMMDD strings. + +### 3. Data Persistence (Optional) + +If your data source supports writing or deleting data, you should also implement: + +- `_write_merged_prices_for_contract_object_no_checking` +- `_write_prices_at_frequency_for_contract_object_no_checking` +- `_delete_merged_prices_for_contract_object_with_no_checks_be_careful` +- `_delete_prices_at_frequency_for_contract_object_with_no_checks_be_careful` + +## Data Format + +The `futuresContractPrices` object is a specialized `pd.DataFrame`. Your implementation must return a DataFrame with the following columns: + +- `OPEN` (float): Opening price. +- `HIGH` (float): High price. +- `LOW` (float): Low price. +- `FINAL` (float): Closing or settlement price (Mandatory). +- `VOLUME` (float): Traded volume. + +The index of the DataFrame must be a `DatetimeIndex` named `index`. + +## Implementation Hint for APIs + +For API-based implementations, it is common to: + +1. Maintain or fetch a mapping of `instrument_code` (e.g., 'CRUDE_OIL') to specific vendor tickers (e.g., 'CL1 Comdty', 'CLM24 Comdty'). +2. Use the `contract_object.date_str` (YYYYMMDD) or `contract_object.instrument` to construct the necessary API request. +3. Handle missing data gracefully by returning `futuresContractPrices.create_empty()`. + +Refer to `csvFuturesContractPriceData` in `csv/csv_futures_contract_prices.py` for a reference implementation using local files. diff --git a/src/quantlib_st/sysdata/futures/futures_per_contract_prices.py b/src/quantlib_st/sysdata/futures/futures_per_contract_prices.py new file mode 100644 index 0000000..107d932 --- /dev/null +++ b/src/quantlib_st/sysdata/futures/futures_per_contract_prices.py @@ -0,0 +1,557 @@ +from abc import abstractmethod, ABC + +from quantlib_st.core.constants import named_object +from quantlib_st.core.exceptions import missingData +from quantlib_st.core.dateutils import Frequency, MIXED_FREQ +from quantlib_st.core.pandas.merge_data_keeping_past_data import ( + SPIKE_IN_DATA, + mergeError, +) + +from quantlib_st.sysdata.base_data import baseData + +from quantlib_st.objects.contracts import futuresContract, listOfFuturesContracts +from quantlib_st.objects.contract_dates_and_expiries import listOfContractDateStr +from quantlib_st.objects.futures_per_contract_prices import futuresContractPrices +from quantlib_st.objects.dict_of_futures_per_contract_prices import ( + dictFuturesContractPrices, +) + +from quantlib_st.logging.logger import * + +BASE_CLASS_ERROR = "You have used a base class for futures price data; you need to use a class that inherits with a specific data source" + +VERY_BIG_NUMBER = 999999.0 + + +#### Three types of price data: at a specific frequency, and 'merged' (no specific frequency, covers all bases) +#### and 'all' (both types) + + +class futuresContractPriceData(baseData, ABC): + """ + Extends the baseData object to a data source that reads in and writes prices for specific futures contracts + + This would normally be extended further for information from a specific source eg quandl, arctic + + Access via: object.get_prices_for_instrumentCode_and_contractDate('EDOLLAR','201702'] + or object.get_prices_for_contract_object(futuresContract(....)) + """ + + def __init__(self, log=get_logger("futuresContractPriceData")): + super().__init__(log=log) + + def __repr__(self) -> str: + return "Individual futures contract price data - DO NOT USE" + + def __getitem__(self, contract_object: futuresContract) -> futuresContractPrices: + """ + convenience method to get the price, make it look like a dict + + """ + + return self.get_merged_prices_for_contract_object(contract_object) + + def keys(self) -> listOfFuturesContracts: + """ + list of things in this data set (futures contracts, instruments...) + + :returns: list of str + + """ + return self.get_contracts_with_merged_price_data() + + def get_list_of_instrument_codes_with_merged_price_data(self) -> list: + """ + + :return: list of str + """ + + list_of_contracts_with_price_data = self.get_contracts_with_merged_price_data() + unique_list_of_instruments = ( + list_of_contracts_with_price_data.unique_list_of_instrument_codes() + ) + + return unique_list_of_instruments + + def get_list_of_instrument_codes_with_price_data_at_frequency( + self, frequency: Frequency + ) -> list: + """ + + :return: list of str + """ + + list_of_contracts_with_price_data = ( + self.get_contracts_with_price_data_for_frequency(frequency=frequency) + ) + unique_list_of_instruments = ( + list_of_contracts_with_price_data.unique_list_of_instrument_codes() + ) + + return unique_list_of_instruments + + def has_merged_price_data_for_contract( + self, contract_object: futuresContract + ) -> bool: + list_of_contracts = self.get_contracts_with_merged_price_data() + if contract_object in list_of_contracts: + return True + else: + return False + + def has_price_data_for_contract_at_frequency( + self, contract_object: futuresContract, frequency: Frequency + ) -> bool: + list_of_contracts = self.get_contracts_with_price_data_for_frequency( + frequency=frequency + ) + if contract_object in list_of_contracts: + return True + else: + return False + + def contracts_with_merged_price_data_for_instrument_code( + self, instrument_code: str + ) -> listOfFuturesContracts: + """ + Valid contracts + + :param instrument_code: str + :return: list of contract_date + """ + + list_of_contracts_with_price_data = self.get_contracts_with_merged_price_data() + list_of_contracts_for_instrument = ( + list_of_contracts_with_price_data.contracts_in_list_for_instrument_code( + instrument_code + ) + ) + + return list_of_contracts_for_instrument + + def contracts_with_price_data_at_frequency_for_instrument_code( + self, instrument_code: str, frequency: Frequency + ) -> listOfFuturesContracts: + """ + Valid contracts + + :param instrument_code: str + :return: list of contract_date + """ + + list_of_contracts_with_price_data = ( + self.get_contracts_with_price_data_for_frequency(frequency=frequency) + ) + list_of_contracts_for_instrument = ( + list_of_contracts_with_price_data.contracts_in_list_for_instrument_code( + instrument_code + ) + ) + + return list_of_contracts_for_instrument + + def contract_dates_with_merged_price_data_for_instrument_code( + self, instrument_code: str + ) -> listOfContractDateStr: + """ + + :param instrument_code: + :return: list of str + """ + + list_of_contracts_with_price_data = ( + self.contracts_with_merged_price_data_for_instrument_code(instrument_code) + ) + + list_of_contract_date_str = list_of_contracts_with_price_data.list_of_dates() + + return list_of_contract_date_str + + def contract_dates_with_price_data_at_frequency_for_instrument_code( + self, instrument_code: str, frequency: Frequency + ) -> listOfContractDateStr: + """ + + :param instrument_code: + :return: list of str + """ + + list_of_contracts_with_price_data = ( + self.contracts_with_price_data_at_frequency_for_instrument_code( + instrument_code=instrument_code, frequency=frequency + ) + ) + + list_of_contract_date_str = list_of_contracts_with_price_data.list_of_dates() + + return list_of_contract_date_str + + def get_merged_prices_for_instrument( + self, instrument_code: str + ) -> dictFuturesContractPrices: + """ + Get all the prices for this code, returned as dict + + :param instrument_code: str + :return: dictFuturesContractPrices + """ + + list_of_contracts = self.contracts_with_merged_price_data_for_instrument_code( + instrument_code + ) + dict_of_prices = dictFuturesContractPrices( + [ + ( + contract.date_str, + self.get_merged_prices_for_contract_object(contract), + ) + for contract in list_of_contracts + ] + ) + + return dict_of_prices + + def get_prices_at_frequency_for_instrument( + self, instrument_code: str, frequency: Frequency + ) -> dictFuturesContractPrices: + """ + Get all the prices for this code, returned as dict + + :param instrument_code: str + :return: dictFuturesContractPrices + """ + + list_of_contracts = ( + self.contracts_with_price_data_at_frequency_for_instrument_code( + instrument_code=instrument_code, frequency=frequency + ) + ) + dict_of_prices = dictFuturesContractPrices( + [ + ( + contract.date_str, + self.get_prices_at_frequency_for_contract_object( + contract, frequency=frequency + ), + ) + for contract in list_of_contracts + ] + ) + + return dict_of_prices + + def get_merged_prices_for_contract_object( + self, contract_object: futuresContract, return_empty: bool = True + ) -> futuresContractPrices: + """ + get all prices without worrying about frequency + + :param contract_object: futuresContract + :return: data + """ + + if self.has_merged_price_data_for_contract(contract_object): + prices = self._get_merged_prices_for_contract_object_no_checking( + contract_object + ) + else: + if return_empty: + return futuresContractPrices.create_empty() + else: + raise missingData + + return prices + + def get_prices_at_frequency_for_contract_object( + self, + contract_object: futuresContract, + frequency: Frequency, + return_empty: bool = True, + ) -> futuresContractPrices: + """ + get some prices at a given frequency + + :param contract_object: futuresContract + :param frequency: str; one of D, H, 5M, M, 10S, S + :return: data + """ + + if self.has_price_data_for_contract_at_frequency( + contract_object, frequency=frequency + ): + return self._get_prices_at_frequency_for_contract_object_no_checking( + contract_object, frequency=frequency + ) + else: + if return_empty: + return futuresContractPrices.create_empty() + else: + raise missingData + + def write_merged_prices_for_contract_object( + self, + futures_contract_object: futuresContract, + futures_price_data: futuresContractPrices | named_object, + ignore_duplication=False, + ): + """ + Write some prices + + :param futures_contract_object: + :param futures_price_data: + :param ignore_duplication: bool, to stop us overwriting existing prices + :return: None + """ + not_ignoring_duplication = not ignore_duplication + if not_ignoring_duplication: + if self.has_merged_price_data_for_contract(futures_contract_object): + self.log.warning( + "There is already existing data for %s" + % futures_contract_object.key, + **futures_contract_object.log_attributes(), + method="temp", + ) + return None + + self._write_merged_prices_for_contract_object_no_checking( + futures_contract_object, futures_price_data + ) + + def write_prices_at_frequency_for_contract_object( + self, + futures_contract_object: futuresContract, + futures_price_data: futuresContractPrices, + frequency: Frequency, + ignore_duplication=False, + ): + """ + Write some prices + + :param futures_contract_object: + :param futures_price_data: + :param ignore_duplication: bool, to stop us overwriting existing prices + :return: None + """ + not_ignoring_duplication = not ignore_duplication + if not_ignoring_duplication: + if self.has_price_data_for_contract_at_frequency( + contract_object=futures_contract_object, frequency=frequency + ): + self.log.warning( + "There is already existing data for %s" + % futures_contract_object.key, + **futures_contract_object.log_attributes(), + method="temp", + ) + return None + + self._write_prices_at_frequency_for_contract_object_no_checking( + futures_contract_object=futures_contract_object, + futures_price_data=futures_price_data, + frequency=frequency, + ) + + def update_prices_at_frequency_for_contract( + self, + contract_object: futuresContract, + new_futures_per_contract_prices: futuresContractPrices, + frequency: Frequency, + check_for_spike: bool = True, + max_price_spike: float = VERY_BIG_NUMBER, + ) -> int: + log_attrs = {**contract_object.log_attributes(), "method": "temp"} + + if len(new_futures_per_contract_prices) == 0: + self.log.debug("No new data", **log_attrs) + return 0 + + if frequency is MIXED_FREQ: + old_prices = self.get_merged_prices_for_contract_object(contract_object) + else: + old_prices = self.get_prices_at_frequency_for_contract_object( + contract_object, frequency=frequency + ) + + merged_prices = old_prices.add_rows_to_existing_data( + new_futures_per_contract_prices, + check_for_spike=check_for_spike, + max_price_spike=max_price_spike, + ) + + if merged_prices is SPIKE_IN_DATA: + self.log.debug( + "Price has moved too much - will need to manually check - no price update done", + **log_attrs, + ) + return SPIKE_IN_DATA + + old_prices = old_prices[~old_prices.index.duplicated(keep="first")] + rows_added = len(merged_prices) - len(old_prices) + + if rows_added < 0: + self.log.critical("Can't remove prices something gone wrong!", **log_attrs) + raise mergeError("Merged prices have fewer rows than old prices!") + + elif rows_added == 0: + if len(old_prices) == 0: + self.log.debug("No existing or additional data", **log_attrs) + return 0 + else: + self.log.debug( + "No additional data since %s " % str(old_prices.index[-1]), + **log_attrs, + ) + return 0 + + # We have guaranteed no duplication + if frequency is MIXED_FREQ: + self.write_merged_prices_for_contract_object( + contract_object, merged_prices, ignore_duplication=True + ) + else: + self.write_prices_at_frequency_for_contract_object( + contract_object, + merged_prices, + frequency=frequency, + ignore_duplication=True, + ) + + self.log.debug("Added %d additional rows of data" % rows_added) + + return rows_added + + def delete_merged_prices_for_contract_object( + self, futures_contract_object: futuresContract, areyousure=False + ): + """ + + :param futures_contract_object: + :return: + """ + + if not areyousure: + raise Exception("You have to be sure to delete prices_for_contract_object!") + + if self.has_merged_price_data_for_contract(futures_contract_object): + self._delete_merged_prices_for_contract_object_with_no_checks_be_careful( + futures_contract_object + ) + else: + self.log.warning( + "Tried to delete non existent contract", + **futures_contract_object.log_attributes(), + method="temp", + ) + + def delete_prices_at_frequency_for_contract_object( + self, + futures_contract_object: futuresContract, + frequency: Frequency, + areyousure=False, + ): + """ + + :param futures_contract_object: + :return: + """ + + if not areyousure: + raise Exception("You have to be sure to delete prices_for_contract_object!") + + if self.has_price_data_for_contract_at_frequency( + futures_contract_object, frequency=frequency + ): + self._delete_prices_at_frequency_for_contract_object_with_no_checks_be_careful( + futures_contract_object=futures_contract_object, frequency=frequency + ) + else: + self.log.warning( + "Tried to delete non existent contract at frequency %s" % frequency, + **futures_contract_object.log_attributes(), + method="temp", + ) + + def delete_merged_prices_for_instrument_code( + self, instrument_code: str, areyousure=False + ): + # We don't pass areyousure, otherwise if we weren't sure would get + # multiple exceptions + if not areyousure: + raise Exception( + "You have to be sure to delete_merged_prices_for_instrument!" + ) + + all_contracts_to_delete = ( + self.contracts_with_merged_price_data_for_instrument_code(instrument_code) + ) + for contract in all_contracts_to_delete: + self.delete_merged_prices_for_contract_object(contract, areyousure=True) + + def delete_prices_at_frequency_for_instrument_code( + self, instrument_code: str, frequency: Frequency, areyousure=False + ): + # We don't pass areyousure, otherwise if we weren't sure would get + # multiple exceptions + if not areyousure: + raise Exception( + "You have to be sure to delete_prices_at_frequency_for_instrument!" + ) + + all_contracts_to_delete = ( + self.contracts_with_price_data_at_frequency_for_instrument_code( + instrument_code=instrument_code, frequency=frequency + ) + ) + + for contract in all_contracts_to_delete: + self.delete_prices_at_frequency_for_contract_object( + futures_contract_object=contract, frequency=frequency, areyousure=True + ) + + @abstractmethod + def get_contracts_with_merged_price_data(self) -> listOfFuturesContracts: + raise NotImplementedError(BASE_CLASS_ERROR) + + @abstractmethod + def get_contracts_with_price_data_for_frequency( + self, frequency: Frequency + ) -> listOfFuturesContracts: + raise NotImplementedError(BASE_CLASS_ERROR) + + def _delete_merged_prices_for_contract_object_with_no_checks_be_careful( + self, futures_contract_object: futuresContract + ): + raise NotImplementedError(BASE_CLASS_ERROR) + + def _delete_prices_at_frequency_for_contract_object_with_no_checks_be_careful( + self, futures_contract_object: futuresContract, frequency: Frequency + ): + raise NotImplementedError(BASE_CLASS_ERROR) + + def _write_merged_prices_for_contract_object_no_checking( + self, + futures_contract_object: futuresContract, + futures_price_data: futuresContractPrices | named_object, + ): + raise NotImplementedError(BASE_CLASS_ERROR) + + def _write_prices_at_frequency_for_contract_object_no_checking( + self, + futures_contract_object: futuresContract, + futures_price_data: futuresContractPrices | named_object, + frequency: Frequency, + ): + raise NotImplementedError(BASE_CLASS_ERROR) + + @abstractmethod + def _get_merged_prices_for_contract_object_no_checking( + self, contract_object: futuresContract + ) -> futuresContractPrices: + raise NotImplementedError(BASE_CLASS_ERROR) + + @abstractmethod + def _get_prices_at_frequency_for_contract_object_no_checking( + self, futures_contract_object: futuresContract, frequency: Frequency + ) -> futuresContractPrices: + raise NotImplementedError(BASE_CLASS_ERROR) diff --git a/src/quantlib_st/sysdata/futures/multiple_prices.py b/src/quantlib_st/sysdata/futures/multiple_prices.py index 31351e3..80a60d6 100644 --- a/src/quantlib_st/sysdata/futures/multiple_prices.py +++ b/src/quantlib_st/sysdata/futures/multiple_prices.py @@ -12,7 +12,6 @@ from quantlib_st.core.exceptions import existingData from quantlib_st.sysdata.base_data import baseData -from quantlib_st.logging.logger import get_logger # These are used when inferring prices in an incomplete series from quantlib_st.objects.multiple_prices import futuresMultiplePrices @@ -48,7 +47,6 @@ def get_multiple_prices(self, instrument_code: str) -> futuresMultiplePrices: return multiple_prices def delete_multiple_prices(self, instrument_code: str, are_you_sure=False): - log_attrs = {INSTRUMENT_CODE_LOG_LABEL: instrument_code, "method": "temp"} if are_you_sure: if self.is_code_in_data(instrument_code): @@ -57,7 +55,8 @@ def delete_multiple_prices(self, instrument_code: str, are_you_sure=False): ) self.log.info( "Deleted multiple price data for %s" % instrument_code, - **log_attrs, + instrument_code=instrument_code, + method="temp", ) else: @@ -65,12 +64,14 @@ def delete_multiple_prices(self, instrument_code: str, are_you_sure=False): self.log.warning( "Tried to delete non existent multiple prices for %s" % instrument_code, - **log_attrs, + instrument_code=instrument_code, + method="temp", ) else: self.log.error( "You need to call delete_multiple_prices with a flag to be sure", - **log_attrs, + instrument_code=instrument_code, + method="temp", ) raise Exception("You need to be sure!") @@ -86,7 +87,6 @@ def add_multiple_prices( multiple_price_data: futuresMultiplePrices, ignore_duplication=False, ): - log_attrs = {INSTRUMENT_CODE_LOG_LABEL: instrument_code, "method": "temp"} if self.is_code_in_data(instrument_code): if ignore_duplication: pass @@ -94,7 +94,8 @@ def add_multiple_prices( self.log.error( "There is already %s in the data, you have to delete it first" % instrument_code, - **log_attrs, + instrument_code=instrument_code, + method="temp", ) raise existingData @@ -102,7 +103,11 @@ def add_multiple_prices( instrument_code, multiple_price_data ) - self.log.info("Added data for instrument %s" % instrument_code, **log_attrs) + self.log.info( + "Added data for instrument %s" % instrument_code, + instrument_code=instrument_code, + method="temp", + ) def _add_multiple_prices_without_checking_for_existing_entry( self, instrument_code: str, multiple_price_data: futuresMultiplePrices diff --git a/src/quantlib_st/sysdata/fx/spotfx.py b/src/quantlib_st/sysdata/fx/spotfx.py index 4e09660..4fa69c1 100644 --- a/src/quantlib_st/sysdata/fx/spotfx.py +++ b/src/quantlib_st/sysdata/fx/spotfx.py @@ -131,7 +131,8 @@ def _get_fx_prices(self, code: str) -> fxPrices: if not self.is_code_in_data(code): self.log.warning( "Currency %s is missing from list of FX data" % code, - **{CURRENCY_CODE_LOG_LABEL: code, "method": "temp"}, + currency_code=code, + method="temp", ) return fxPrices.create_empty() @@ -141,26 +142,28 @@ def _get_fx_prices(self, code: str) -> fxPrices: return data def delete_fx_prices(self, code: str, are_you_sure=False): - log_attrs = {CURRENCY_CODE_LOG_LABEL: code, "method": "temp"} if are_you_sure: if self.is_code_in_data(code): self._delete_fx_prices_without_any_warning_be_careful(code) self.log.info( "Deleted fx price data for %s" % code, - **log_attrs, + currency_code=code, + method="temp", ) else: # doesn't exist anyway self.log.warning( "Tried to delete non existent fx prices for %s" % code, - **log_attrs, + currency_code=code, + method="temp", ) else: self.log.warning( "You need to call delete_fx_prices with a flag to be sure", - **log_attrs, + currency_code=code, + method="temp", ) def is_code_in_data(self, code: str) -> bool: @@ -172,7 +175,6 @@ def is_code_in_data(self, code: str) -> bool: def add_fx_prices( self, code: str, fx_price_data: fxPrices, ignore_duplication: bool = False ): - log_attrs = {CURRENCY_CODE_LOG_LABEL: code, "method": "temp"} if self.is_code_in_data(code): if ignore_duplication: pass @@ -180,12 +182,15 @@ def add_fx_prices( self.log.warning( "There is already %s in the data, you have to delete it first, or " "set ignore_duplication=True, or use update_fx_prices" % code, - **log_attrs, + currency_code=code, + method="temp", ) return None self._add_fx_prices_without_checking_for_existing_entry(code, fx_price_data) - self.log.info("Added fx data for code %s" % code, **log_attrs) + self.log.info( + "Added fx data for code %s" % code, currency_code=code, method="temp" + ) def update_fx_prices( self, code: str, new_fx_prices: fxPrices, check_for_spike=True diff --git a/src/quantlib_st/sysdata/sim/futures_sim_data.py b/src/quantlib_st/sysdata/sim/futures_sim_data.py index 0fdcf5c..a03c889 100644 --- a/src/quantlib_st/sysdata/sim/futures_sim_data.py +++ b/src/quantlib_st/sysdata/sim/futures_sim_data.py @@ -117,7 +117,7 @@ def get_instrument_raw_carry_data(self, instrument_code: str) -> pd.DataFrame: def get_current_and_forward_price_data(self, instrument_code: str) -> pd.DataFrame: """ - Returns a pd. dataframe with the 4 columns PRICE, PRICE_CONTRACT, FORWARD_, FORWARD_CONTRACT + Returns a pd. dataframe with the 4 columns PRICE, PRICE_CONTRACT, FORWARD, FORWARD_CONTRACT These are required if we want to backadjust from scratch