#region imports
from AlgorithmImports import *
import pandas as pd
import datetime
import pickle

#endregion
class ParticleTransdimensionalAutosequencers(QCAlgorithm):
    def Initialize(self):
        
        # Original variables
        # BASIC SETTINGS
        self.SetStartDate(2023, 1, 13)
        self.SetEndDate(2023, 1, 13)
        #self.SetEndDate(2023, 11, 26)
        self.SetCash(20000)
        self.cash = 20000
        self.AddEquity('SPY', Resolution.Minute, extendedMarketHours=True)
        self.market = 'SPY'
        self.SetUniverseSelection(ScheduledUniverseSelectionModel(
            self.DateRules.EveryDay(self.market),
            self.TimeRules.At(9, 0),
            self.SelectSymbols # selection function in algorithm.
        ))
        #self.AddUniverseSelection(CoarseFundamentalUniverseSelectionModel(self.CoarseSelectionFunction))
        self.UniverseSettings.Resolution = Resolution.Second
        algorithm = self
        self.weight = 1
        self.symbols = []
        self.symbols_obj = []
        self.symbolData = {}
        self.consol_dict = {}
        self.gap ={}
        self.gap_neg = {}
        self.gap_combined = {}
        self.pm_range = {}
        self.TrackOpen_hour = 9
        self.TrackOpen_min = 31
        self.levels_lookback_PM = 1
        self.levels_lookforward_PM = 1
        self.support = {}
        self.support_levels = {}
        self.resistance = {}
        self.resistance_levels = {}
        self.all_levels = {}
        self.rvol_PM = {}
        self.can_trade_flag = False
        self.long_trade_flag = False
        self.short_trade_flag = False
        self.min_price = 10
        self.RRR = 1.0
        self.min_spread = 0.1
        self.min_profit = 100
        self.risk_per_trade = 100
        self.entry_orders = {}
        self.take_profit = {}
        self.stop_loss = {}
        self.SL = {}
        self.SL_id = {}
        self.TP = {}
        self.trades = {}
        self.entry_prices = {}
        self.daily_PL = 0
        self.highest_low = {}
        self.lowest_high = {}
        self.reached_tp = {}
        self.winning_trades = 0
        self.winning_PL = 0
        self.losing_trades = 0
        self.losing_PL = 0
        # for dataframe creation
        self.df_store = pd.DataFrame([],columns = ['time','symbol', 'gap_perc','pm_rvol','pm-low','pm_s1','pm_s2', 'pm_s3','pm_s1_rvol','pm_s2_rvol','pm_s3_rvol','pm-high','pm_r1','pm_r2','pm_r3','pm_r1_rvol','pm_r2_rvol','pm_r3_rvol'])
        self.df_store_day = pd.DataFrame([],columns = ['time','symbol', 'gap_perc','pm_rvol','pm-low','pm_s1','pm_s2', 'pm_s3','pm_s1_rvol','pm_s2_rvol','pm_s3_rvol','pm-high','pm_r1','pm_r2','pm_r3','pm_r1_rvol','pm_r2_rvol','pm_r3_rvol'])
        self.Schedule.On(self.DateRules.EveryDay(self.market), self.TimeRules.At(8, 59), self.Clear_Consol)
        self.Schedule.On(self.DateRules.EveryDay(self.market), self.TimeRules.At(9, 1), self.Add_Consol)
        self.Schedule.On(self.DateRules.EveryDay(self.market), self.TimeRules.At(9, 31, 0), self.AtOpen)
        self.Schedule.On(self.DateRules.EveryDay(self.market), self.TimeRules.At(9, 45), self.ClosePositions)
        self.day_count = 0
        self.project_key = "15610290/PM_data_2023-1-1_227 days out_2"

    # UNIVERSE SELECTION (09:30)
    # Create selection function which returns symbol objects.
    def SelectSymbols(self, dateTime):
        self.Debug(f"Enter Scheduled universe at {self.Time}. Universe update.")
        self.symbols = []
        self.symbols_obj = []
        self.volume_by_symbol = {}
        self.daily_PL = 0
        adjusted_time = self.Time.replace(hour=9, minute=31, second=0)
        dict_of_symbols = symbol_extractor(self.project_key, self)
        tickers = dict_of_symbols[adjusted_time]
        tickers_symbol = [Symbol.Create(ticker, SecurityType.Equity, Market.USA) for ticker in tickers]
        self.symbols_obj = tickers_symbol           
        self.volume_by_symbol = {c: 0 for c in tickers_symbol}
        self.highest_low = {c: 0 for c in tickers_symbol}
        self.lowest_high = {c: 10000000 for c in tickers_symbol}
        self.symbols = list(self.volume_by_symbol.keys())
        self.Debug(f"Universe size before volume filter: {len(self.volume_by_symbol)} at {self.Time}")
        return list(self.volume_by_symbol.keys())

    def OnDataConsolidated(self, sender, bar):
        # self.Debug(f'Information for {sender.Consolidated.Symbol} - Open: {bar.Open}, Close: {bar.Close}, Period: {bar.Period} at {self.Time}')
        symbol = sender.Consolidated.Symbol
        # Take profit or loss after stop time is reached
        stop_time = self.Time.replace(hour=9, minute=45, second=0)
        if self.Time > stop_time:
            return
        # Track if stocks are tradable at that time range
        start_time = self.Time.replace(hour=9, minute=31, second=0)
        end_time = self.Time.replace(hour=9, minute=33, second=0)
        if self.Time >= start_time and self.Time <= end_time :
            self.can_trade_flag = True
        else:
            self.can_trade_flag = False
        if self.can_trade_flag == True:
            # reset trade flags before running through the trade conditions
            self.long_trade_flag = False
            self.short_trade_flag = False            
            # CHECK FOR ENTRY CONDITIONS
            # Basic buy and sell condtions to meet
            pm_h = self.df_store_day.loc[symbol,'pm-high']
            pm_l = self.df_store_day.loc[symbol,'pm-low']
            up_cross = bar.High > pm_h and bar.Low < pm_h
            down_cross = bar.Low < pm_l and bar.High > pm_l
            green_bar = bar.Close > bar.Open
            red_bar = bar.Close < bar.Open
            long_condition = up_cross and green_bar
            short_condition = down_cross and red_bar
            min_price_condition = bar.Low > self.min_price
            # Further risk reward conditions to meet
            if long_condition and min_price_condition:
                #self.Debug(f'{symbol} met the long condition at {self.Time}')
                # When stocks met long criteria and min_price_condition, check if risk reward makes sense
                # Metrics to calculate take profit and stop loss
                start_time = self.Time.replace(hour=9, minute=30, second=0)
                actual_BS_time = self.Time
                pre_trade_history = Consolidator_2(self, symbol.Value, start_time, actual_BS_time, base_freq = Resolution.Second, consol_freq = 10)
                low_since_open = np.min(pre_trade_history['low'])
                high_since_open = np.max(pre_trade_history['high'])
                # Calculate atr as metric for take profit and stop loss
                Sym = self.AddEquity(symbol.Value)
                #atr_sym = self.ATR(self.symbol, 50, MovingAverageType.Simple)
                #atr_history = self.Indicator(atr, Sym.Symbol, actual_BS_time - timedelta(days=50), actual_BS_time, Resolution.Daily)
                atr = AverageTrueRange(20)
                history = self.History(Sym.Symbol,50,Resolution.Daily)
                for bar_1 in history.itertuples():
                    tradebar = TradeBar(bar_1.Index[1], Sym.Symbol, bar_1.open, bar_1.high, bar_1.low, bar_1.close, bar_1.volume)
                    atr.Update(tradebar)
                atr_sym = atr.Current.Value
                range_at_open = atr_sym * 0.3
                # Calculate stop-loss & risk
                theoretical_trade_price = pre_trade_history.iloc[-1].high
                stop_loss_low = low_since_open
                stop_loss_atr = theoretical_trade_price - range_at_open * 0.5
                stop_loss = max(stop_loss_low, stop_loss_atr)
                risk = theoretical_trade_price - stop_loss
                # Calculate take-profit
                take_profit_RRR = theoretical_trade_price + self.RRR * risk
                take_profit_atr = theoretical_trade_price + range_at_open * 0.5
                take_profit = max(take_profit_RRR, take_profit_atr)   
                # Calculate pre-trade volume
                pre_trade_volume = np.mean(pre_trade_history.volume)
                # Calculate spread
                spread = abs(take_profit - theoretical_trade_price)
                min_trade_volume = self.min_profit / (spread * bar.Low)           
                if spread >= self.min_spread and pre_trade_volume * 0.25 > min_trade_volume:
                    self.long_trade_flag = True
                    self.Debug(f'{symbol} met the long trade condition at {self.Time}')
                else:
                    self.long_trade_flag = False
                    self.Debug(f'{symbol} met the long & min_price conditions but not long trade condtion at {self.Time}')
            elif short_condition and min_price_condition:
                #self.Debug(f'{symbol} met the short condition at {self.Time}')
                start_time = self.Time.replace(hour=9, minute=30, second=0)
                actual_BS_time = self.Time
                pre_trade_history = Consolidator_2(self, symbol.Value, start_time, actual_BS_time, base_freq = Resolution.Second, consol_freq = 10)
                low_since_open = np.min(pre_trade_history['low'])
                high_since_open = np.max(pre_trade_history['high'])
                # Calculate atr as metric for take profit and stop loss
                Sym = self.AddEquity(symbol.Value)
                atr = AverageTrueRange(20)
                #atr_sym = self.ATR(self.symbol, 50, MovingAverageType.Simple)
                #atr_history = self.Indicator(atr, Sym.Symbol, actual_BS_time - timedelta(days=50), actual_BS_time, Resolution.Daily)
                #atr_sym = atr_history.iloc[-1]['averagetruerange']
                history = self.History(Sym.Symbol,50,Resolution.Daily)
                for bar_1 in history.itertuples():
                    tradebar = TradeBar(bar_1.Index[1], Sym.Symbol, bar_1.open, bar_1.high, bar_1.low, bar_1.close, bar_1.volume)
                    atr.Update(tradebar)
                atr_sym = atr.Current.Value
                range_at_open = atr_sym * 0.3
                # Calculate stop-loss & risk
                theoretical_trade_price = pre_trade_history.iloc[-1].low
                stop_loss_high = high_since_open
                stop_loss_atr = theoretical_trade_price + range_at_open * 0.5
                stop_loss = min(stop_loss_high, stop_loss_atr)
                risk = stop_loss - theoretical_trade_price
                # Calculate take-profit
                take_profit_RRR = theoretical_trade_price - self.RRR * risk
                take_profit_atr = theoretical_trade_price - range_at_open * 0.5
                take_profit = min(take_profit_RRR, take_profit_atr)   
                # Calculate pre-trade volume
                pre_trade_volume = np.mean(pre_trade_history.volume)
                # Calculate spread
                spread = abs(take_profit - theoretical_trade_price)
                min_trade_volume = self.min_profit / (spread * bar.Low)           
                if spread >= self.min_spread and pre_trade_volume * 0.25 > min_trade_volume:
                    self.short_trade_flag = True
                    #self.Debug(f'{symbol} met the trade condition at {self.Time}')
                    self.Debug(f'{symbol} met the short trade condition at {self.Time}')
                else:
                    self.short_trade_flag = False
                    self.Debug(f'{symbol} met the short & min_price conditions but not short trade condtion at {self.Time}')
            
            # If the instance meets all criteria, build the feature dataframe, run it through the ML model, if the ML model returns positive, trade, if not, don't trade
            if (self.long_trade_flag == True) or (self.short_trade_flag == True):
                feature_df_long = pd.DataFrame([],columns = ['gap_pct', 'pm_rvol', 'pm_range', 'time_of_trade', 'bid_ask_latest', 'max_vol_pre_trade', 'latest_vol_pre_trade', 'volume_pre_trade', 'volume_pre_trade_pct', 'volume_trend', 'open_above_r3', 'open_above_pm_high', 'pct_green_vol', 'volume_intensity', 'pct_volume_above_pm_high', 'dist_r3_high'])
                feature_df_short = pd.DataFrame([],columns = ['gap_pct', 'pm_rvol', 'pm_range', 'time_of_trade', 'bid_ask_latest','max_vol_pre_trade', 'latest_vol_pre_trade', 'volume_pre_trade', 'volume_pre_trade_pct', 'volume_trend', 'open_below_s3', 'pct_green_vol', 'volume_intensity', 'pct_volume_below_pm_low', 'dist_s3_low'])
                # Gap Percentage
                gap_pct = self.gap_combined[symbol]
                # Relative volume in PM (PM volume as percentage of average daily volume in the past 20 days)
                pm_rvol = self.rvol_PM[symbol]
                # Relative Volitilty in PM (PM range as percentage of ATR)
                pm_range = self.pm_range[symbol]
                pm_range_as_pct = pm_range / atr_sym
                # time of trade. How many bars has gone by before meeting the trade criteria
                BS_time = bar.Time
                BS_min = BS_time.minute
                BS_sec = BS_time.second
                no_of_bar = (BS_min - 30) * 6 + BS_sec / 10
                # latest bid ask spread before having a trade
                ask_close = pre_trade_history['askclose']
                bid_close = pre_trade_history['bidclose']
                bid_ask_spread = ask_close - bid_close
                latest_bid_ask = bid_ask_spread.iloc[-1]
                # Maximum volatility and latest volatility pre trade
                volatility = pre_trade_history['high'] - pre_trade_history['low']
                latest_vol = volatility.iloc[-1] / atr_sym
                max_vol = max(volatility) / atr_sym
                # volume after open before trade as % of avg daily volume
                start_d = BS_time - timedelta(days=20)
                stop_d = BS_time
                history_d = self.History(Sym.Symbol, start_d, stop_d, Resolution.Daily, extendedMarketHours=False)
                symbol_sma = np.mean(history_d['volume'].values)
                volume = pre_trade_history['volume']
                pre_trade_vol = sum(volume)
                pre_trade_vol_as_pct = pre_trade_vol / symbol_sma
                # volume trend after open before trade
                volume_trend = find_slope(volume)
                # stock open above r3 and below pm-high
                stock_open = pre_trade_history['open'][0]
                pm_high = self.df_store_day.loc[symbol,'pm-high']
                pm_low = self.df_store_day.loc[symbol,'pm-low']
                pm_r3 = max(self.df_store_day.loc[symbol,'pm_r1'], self.df_store_day.loc[symbol,'pm_r2'], self.df_store_day.loc[symbol,'pm_r3'])
                if (stock_open > pm_r3) and (stock_open < pm_high):
                    above_r3 = 1
                else:
                    above_r3 = 0
                # stock open above pm-high
                if stock_open > pm_high:
                    above_pm_high = 1
                else:
                    above_pm_high = 0
                # stock open below pm-low
                if stock_open < pm_low:
                    below_pm_low = 1
                else:
                    below_pm_low = 0
                # stock open below s3 and above pm-low
                pm_support = [self.df_store_day.loc[symbol,'pm_s1'], self.df_store_day.loc[symbol,'pm_s2'], self.df_store_day.loc[symbol,'pm_s3']]
                pm_suppport_filtered = [x for x in pm_support if x !=0]
                try:
                    pm_s3 = min(pm_suppport_filtered)
                    if (stock_open < pm_s3) and (stock_open > pm_low):
                        below_s3 = 1
                    else:
                        below_s3 = 0
                except:
                    below_s3 = 0
                # total green volume vs red volume
                condition_g = (pre_trade_history['close'] > pre_trade_history['open'])
                total_green_volumes = pre_trade_history.loc[condition_g, 'volume'].sum()
                condition_r = (pre_trade_history['close'] < pre_trade_history['open'])
                total_red_volumes = pre_trade_history.loc[condition_r, 'volume'].sum()
                pct_green_vol = total_green_volumes / (total_green_volumes + total_red_volumes)
                # The most recent volume bar as % of all volume since open
                most_recent_two_bar_volume = pre_trade_history.volume.iloc[-1] + pre_trade_history.volume.iloc[-2]
                total_vol_since_open = pre_trade_history.volume.sum()
                pct_recent_bars_volume = most_recent_two_bar_volume / total_vol_since_open
                # % of volume below PM-low (for sell) and above PM-high (for buy) before trade
                condition_pm_high = (pre_trade_history['close'] > self.df_store_day.loc[symbol,'pm-high'])
                total_pm_high_volumes = pre_trade_history.loc[condition_pm_high, 'volume'].sum()
                pct_pm_high_vol = total_pm_high_volumes / total_vol_since_open
                condition_pm_low = (pre_trade_history['close'] < self.df_store_day.loc[symbol,'pm-low'])
                total_pm_low_volumes = pre_trade_history.loc[condition_pm_low, 'volume'].sum()
                pct_pm_low_vol = total_pm_low_volumes / total_vol_since_open
                # distance between r3 vs high and s3 vs low as % of atr
                distance_r3_high = pm_high - pm_r3
                distance_s3_low =  pm_s3 - pm_low
                pct_dist_r3_high = distance_r3_high / atr_sym
                pct_dist_s3_low = distance_s3_low / atr_sym                               
                # Feed into pre-trained model to see what is the prediction
                Long_serialized_obj = bytes(self.ObjectStore.ReadBytes("OD_Long_AB_model"))
                Long_model = pickle.loads(Long_serialized_obj)
                Short_serialized_obj = bytes(self.ObjectStore.ReadBytes("OD_Short_GB_model"))
                Short_model = pickle.loads(Short_serialized_obj)
                if self.long_trade_flag == True:
                    # Enter that into the dataframe
                    feature_df_long.loc[len(feature_df_long)] = [gap_pct, pm_rvol, pm_range_as_pct, no_of_bar, latest_bid_ask, max_vol, latest_vol, pre_trade_vol, pre_trade_vol_as_pct, volume_trend, above_r3, above_pm_high, pct_green_vol, pct_recent_bars_volume, pct_pm_high_vol, pct_dist_r3_high]
                    feature_df_long_no_na = feature_df_long.dropna()
                    if feature_df_long_no_na.empty:
                        yhat_pred = 0
                    else:
                        yhat_pred = Long_model.predict(feature_df_long)
                    if yhat_pred > 0:
                        self.Debug(f'yhat for {symbol} is {yhat_pred}, Buy the stock.')
                        # Quantity
                        risk_per_trade = self.risk_per_trade
                        theoretical_quantity = risk_per_trade / risk
                        buying_power = self.Portfolio.GetBuyingPower(symbol, OrderDirection.Buy)
                        binding_quantity = buying_power / (bar.High * 1.02)
                        quantity = min(theoretical_quantity, binding_quantity)
                        # Market order
                        self.entry_orders[(symbol,self.Time)] = (True,"Buy")
                        self.take_profit[(symbol,self.Time)] = take_profit
                        self.stop_loss[(symbol, self.Time)] = stop_loss
                        if symbol in self.trades:
                            self.trades[symbol].append((symbol,self.Time))
                        else:
                            self.trades[symbol] = [(symbol,self.Time)]
                        self.MarketOrder(symbol, quantity)
                        self.reached_tp[(symbol, self.Time)] = False 
                    else:
                        self.Debug(f'yhat for {symbol} is {yhat_pred}, Dont buy the stock even if it meets all the criteria.')
                elif self.short_trade_flag == True:
                    feature_df_short.loc[len(feature_df_short)] = [gap_pct, pm_rvol, pm_range_as_pct, no_of_bar, latest_bid_ask, max_vol, latest_vol, pre_trade_vol, pre_trade_vol_as_pct, volume_trend, below_s3, pct_green_vol, pct_recent_bars_volume, pct_pm_low_vol, pct_dist_s3_low]
                    feature_df_short_no_na = feature_df_short.dropna()
                    if feature_df_short_no_na.empty:
                        yhat_pred = 0
                    else:
                        yhat_pred = Short_model.predict(feature_df_short)
                    if yhat_pred > 0:
                        self.Debug(f'yhat for {symbol} is {yhat_pred}, Short the stock.')
                        # Quantity
                        risk_per_trade = self.risk_per_trade
                        theoretical_quantity = risk_per_trade / risk
                        buying_power = self.Portfolio.GetBuyingPower(symbol, OrderDirection.Sell)
                        binding_quantity = buying_power / (bar.Low * 1.02)
                        quantity = min(theoretical_quantity, binding_quantity)
                        # Market order
                        self.entry_orders[(symbol,self.Time)] = (True, "Sell")
                        self.take_profit[(symbol,self.Time)] = take_profit
                        self.stop_loss[(symbol, self.Time)] = stop_loss
                        if symbol in self.trades:
                            self.trades[symbol].append((symbol,self.Time))
                        else:
                            self.trades[symbol] = [(symbol,self.Time)]
                        self.MarketOrder(symbol, -quantity)
                        self.reached_tp[(symbol, self.Time)] = False                       
                    else:
                        self.Debug(f'yhat for {symbol} is {yhat_pred}, Dont short the stock even if it meets all the criteria.')
        # MONITOR ALL OPEN TRADES BETWEEN 9:30 - 9:45 TO UPDATE ITS STOP PRICE AT THE RIGHT TIME
        liquidate_time = self.Time.replace(hour=9, minute=45, second=0)
        if self.Time >= start_time and self.Time <= liquidate_time:
            # Update highest_low and lowest_high
            self.highest_low[symbol] = max(bar.Low, self.highest_low[symbol])
            self.lowest_high[symbol] = min(bar.High, self.lowest_high[symbol])
            if symbol in self.trades:
                trades = self.trades[symbol]
                for trade in trades:
                    buy_sell = self.entry_orders[trade][1]
                    if buy_sell == "Buy":
                        TP = self.take_profit[trade]
                        latest_high = bar.High
                        highest_low = self.highest_low[symbol]
                        if latest_high >= TP and self.reached_tp[trade] == False:
                            self.reached_tp[trade] = True
                            self.Debug(f"{symbol} reached target price at {self.Time}")
                        if self.reached_tp[trade] == True:
                            revised_stop = max(TP, highest_low)
                            ticket = self.SL[trade]
                            update_settings = UpdateOrderFields()
                            update_settings.StopPrice = revised_stop
                            response = ticket.Update(update_settings)
                            self.Debug(f"Stop Loss update attempted for {symbol} long trade {trade}")
                            if response.IsSuccess:
                                self.Debug(f"Order updated successfully. {symbol} long trade {trade} stop loss updated to {revised_stop}")
                    elif buy_sell == "Sell":
                        TP = self.take_profit[trade]
                        latest_low = bar.Low
                        lowest_high = self.lowest_high[symbol]
                        if latest_low <= TP and self.reached_tp[trade] == False:
                            self.reached_tp[trade] = True
                            self.Debug(f"{symbol} reached target price at {self.Time}")
                        if self.reached_tp[trade] == True:
                            revised_stop = min(TP, lowest_high)
                            ticket = self.SL[trade]
                            update_settings = UpdateOrderFields()
                            update_settings.StopPrice = revised_stop
                            response = ticket.Update(update_settings)
                            self.Debug(f"Stop Loss update attempted for {symbol} short trade {trade}")
                            if response.IsSuccess:
                                self.Debug(f"Order updated successfully. {symbol} short trade {trade} stop loss updated to {revised_stop}")                            

    def OnOrderEvent(self, orderEvent):
        if orderEvent.Status == OrderStatus.Filled:
            if (orderEvent.Symbol, self.Time) in self.entry_orders:
                stop_loss = self.stop_loss[(orderEvent.Symbol, self.Time)]
                take_profit = self.take_profit[(orderEvent.Symbol, self.Time)]                
                # For buy orders
                if orderEvent.FillQuantity > 0:
                    profit_potential = take_profit - orderEvent.FillPrice
                    risk = orderEvent.FillPrice - stop_loss
                    implied_RRR = profit_potential / risk
                    self.Debug(f"BUY {orderEvent.Symbol} at {self.Time} with market entry at {orderEvent.FillPrice}, stop loss at {stop_loss}, RRR at {implied_RRR}, risk at {risk}")
                # For sell orders
                elif orderEvent.FillQuantity < 0:
                    profit_potential = orderEvent.FillPrice - take_profit
                    risk = stop_loss - orderEvent.FillPrice
                    implied_RRR = profit_potential / risk
                    self.Debug(f"SELL {orderEvent.Symbol} at {self.Time} with market entry at {orderEvent.FillPrice}, stop loss at {stop_loss}, RRR at {implied_RRR}, risk at {risk}")
                self.SL[(orderEvent.Symbol, self.Time)] = self.StopMarketOrder(orderEvent.Symbol, -orderEvent.FillQuantity, stop_loss)
                self.SL_id[(orderEvent.Symbol, self.Time)] = self.SL[(orderEvent.Symbol, self.Time)].OrderId
                self.entry_prices[(orderEvent.Symbol, self.Time)] = orderEvent.FillPrice
            else:
                list_of_ids = [key for key, val in self.SL_id.items() if val == orderEvent.OrderId]
                if len(list_of_ids) == 0:
                    self.Debug(f"Liquidated {orderEvent.Symbol} at {self.Time}")
                    exit_price = orderEvent.FillPrice
                    ls_of_outstanding_trades = self.trades[orderEvent.Symbol]
                    for trade in ls_of_outstanding_trades:
                        entry_price = self.entry_prices[trade]
                        PL = orderEvent.FillQuantity * (entry_price - orderEvent.FillPrice)
                        if PL >= 0:
                            self.Debug(f"Take Profit at {orderEvent.FillPrice} and won {PL} for {orderEvent.Symbol} {trade}.")
                            self.winning_PL += PL
                            self.winning_trades += 1
                        elif PL < 0:
                            self.Debug(f"Take Loss at {orderEvent.FillPrice} and lost {PL} for {orderEvent.Symbol} {trade}.")
                            self.losing_PL += PL
                            self.losing_trades += 1
                        self.daily_PL += PL
                        self.Debug(f"Daily P&L is now {self.daily_PL} at {self.Time}")                         
                else:
                    trade_asso_with_SL = list_of_ids[0]
                    entry_price = self.entry_prices[trade_asso_with_SL]
                    PL = orderEvent.FillQuantity * (entry_price - orderEvent.FillPrice)
                    if PL >= 0:
                        self.Debug(f"Take Profit at {orderEvent.FillPrice} and won {PL} for {orderEvent.Symbol} {trade_asso_with_SL}.")
                        self.winning_PL += PL
                        self.winning_trades += 1
                    elif PL < 0:
                        self.Debug(f"Take Loss at {orderEvent.FillPrice} and lost {PL} for {orderEvent.Symbol} {trade_asso_with_SL}.")
                        self.losing_PL += PL
                        self.losing_trades += 1                        
                    self.daily_PL += PL
                    self.Debug(f"Daily P&L is now {self.daily_PL} at {self.Time}")
                    self.trades[orderEvent.Symbol].remove(trade_asso_with_SL)
                    if len(self.trades[orderEvent.Symbol]) == 0:
                        self.trades.pop(orderEvent.Symbol)

    def Clear_Consol(self):
        self.Debug(f"Clear consol dict for previous day at {self.Time} before universe update")
        for removed in self.consol_dict.items():
            if removed is not None:
                self.SubscriptionManager.RemoveConsolidator(removed[1].symbol, removed[1].TenSecConsolidator)
    
    def Add_Consol(self):
        self.Debug(f"Add to consol dict for current day at {self.Time} after universe update")
        for symbol in self.symbols:
            self.consol_dict[symbol] = SymbolData(symbol, self)
            self.consol_dict[symbol].TenSecConsolidator.DataConsolidated += self.OnDataConsolidated
            self.SubscriptionManager.AddConsolidator(symbol, self.consol_dict[symbol].TenSecConsolidator)    
        
    def AtOpen(self):
        self.Debug(f'Going into AtOpen at {self.Time}, active universe has {len(self.ActiveSecurities)} number of symbols')
        self.Debug(f'Going into AtOpen at {self.Time}, self.symbols has {len(self.symbols)} number of symbols')
        # Track gap% 
        open_by_symbol = {}
        gap_thres = 0.03
        # Empty dictionaries / dataframe from previous day
        self.gap.clear()
        self.gap_neg.clear()
        self.gap_combined.clear()
        self.rvol_PM.clear()
        self.support.clear()
        self.support_levels.clear() 
        self.resistance.clear()
        self.resistance_levels.clear()
        self.all_levels.clear()
        self.pm_range.clear()
        self.Debug(f"dictionaries cleared before a new trading day")

        for symbol in self.symbols:
            #start = self.Time.replace(hour=9, minute=30, second=0)
            #stop = self.Time
            #historyDataMin = self.History(symbol, start, stop, Resolution.Second, extendedMarketHours=True)
            historyDataMin = self.History(symbol,1,Resolution.Minute)
            #historyDataMin = self.History(symbol,1,Resolution.Second)
            try:
                open_price_sym = historyDataMin['open'][-1]
                #open_price_sym = historyDataMin['open'][0]
                #self.Debug(f"Opening price for {symbol.Value} is {open_price_sym}")
            except:
                self.Debug(f"Opening price data for current day unavailable for {symbol.Value}")
                open_price_sym = 0
            open_by_symbol[symbol] = open_price_sym

            historyData = self.History(symbol,2,Resolution.Daily)
            try:
                closeDayBefore = historyData['close'][-1]
                #self.Debug(f"Close price the day before for {symbol.Value} is {closeDayBefore}")
            except:
                self.Debug(f"History data unavailable for {symbol.Value}")
                continue
            priceGap = open_by_symbol[symbol] - closeDayBefore
            percentGap = priceGap / closeDayBefore
            #self.Debug(f"Percent gap for {symbol.Value} is {percentGap}")
            if (percentGap > gap_thres):
                self.gap[symbol] = percentGap
            if (percentGap < -gap_thres):
                self.gap_neg[symbol] = percentGap
        
        self.gap_combined = {**self.gap , **self.gap_neg}
        self.Debug(f'Universe size after gap positive filter: {len(self.gap)} and negative filter: {len(self.gap_neg)} at {self.Time}')

        # Track PM Rvol 
        for symbol, gap in self.gap_combined.items():
            previous_day = self.Time - datetime.timedelta(days=1)
            start = previous_day.replace(hour=16, minute=0, second=0)
            stop = self.Time.replace(hour=9, minute=30, second=0)
            history = self.History(symbol, start, stop, Resolution.Minute, extendedMarketHours=True)
            try:
                PM_vol_total = np.nansum(history['volume'].values)
            except:
                PM_vol_total = 0
            previous_20_day = self.Time - datetime.timedelta(days=20)
            start_d = previous_20_day.replace(hour=9, minute=30, second=0)
            stop_d = self.Time            
            history_d = self.History(symbol, start_d, stop_d, Resolution.Daily, extendedMarketHours=False)
            symbol_sma = np.mean(history_d['volume'].values)
            self.rvol_PM[symbol] = PM_vol_total / symbol_sma
            #self.Debug(f'{symbol} has PM rvol of {self.rvol_PM[symbol]}')

        # Track PM Levels
        for symbol, gap in self.gap_combined.items():
        #try:
            ss = {}
            ss_levels = {}
            rr = {}
            rr_levels = {}
            all_levels = {}
            previous_day = self.Time - datetime.timedelta(days=1)
            start = previous_day.replace(hour=16, minute=0, second=0)
            #start = self.Time.replace(hour=8, minute=30, second=0)
            stop = self.Time.replace(hour=9, minute=30, second=0)
            history = self.History(symbol, start, stop, Resolution.Minute, extendedMarketHours=True)
            # history = self.History(symbol, 630 , Resolution.Minute, extendedMarketHours=True)
            try:
                pm_high = history['high'].max()
            except:
                pm_high = 100000
            try:
                pm_low = history['low'].min()
            except:
                pm_low = 0
            n1 = self.levels_lookback_PM
            n2 = self.levels_lookforward_PM
            l = len(history)
            for row in range(n1, l-n2):
                if support(history,row,n1,n2):
                    ss[history.index[row-1][1]] = history.low[row]
                    rounded_level = round(history.low[row],0)
                    if rounded_level not in list(ss_levels.keys()):
                        try:
                            ss_levels[rounded_level] = {history.low[row]: history.volume[row]}
                        except:
                            self.Debug(f"volume not available for {symbol} at {self.Time}")
                            ss_levels[rounded_level] = {history.low[row]: 0}
                    else:
                        previous_walvl = list(ss_levels[rounded_level].keys())[0]
                        previous_vol = list(ss_levels[rounded_level].values())[0]
                        new_lvl_input = history.low[row]
                        try:
                            new_lvl_vol = history.volume[row]
                        except:
                            new_lvl_vol = 0
                        if (previous_vol + new_lvl_vol) == 0:
                            updated_walvl = (previous_walvl + new_lvl_input) / 2
                        else:
                            updated_walvl = (previous_walvl * previous_vol + new_lvl_input * new_lvl_vol) / (previous_vol + new_lvl_vol)
                        updated_vol = previous_vol + new_lvl_vol
                        ss_levels[rounded_level] = {updated_walvl: updated_vol}
                    if rounded_level not in list(all_levels.keys()):
                        try:
                            all_levels[rounded_level] = {history.low[row]: history.volume[row]}
                        except:
                            self.Debug(f"volume not available for {symbol} at {self.Time}")
                            all_levels[rounded_level] = {history.low[row]: 0}
                    else:
                        previous_walvl = list(all_levels[rounded_level].keys())[0]
                        previous_vol = list(all_levels[rounded_level].values())[0]
                        new_lvl_input = history.low[row]
                        try:
                            new_lvl_vol = history.volume[row]
                        except:
                            new_lvl_vol = 0
                        if (previous_vol + new_lvl_vol) == 0:
                            updated_walvl = (previous_walvl + new_lvl_input) / 2
                        else:
                            updated_walvl = (previous_walvl * previous_vol + new_lvl_input * new_lvl_vol) / (previous_vol + new_lvl_vol)
                        updated_vol = previous_vol + new_lvl_vol
                        all_levels[rounded_level] = {updated_walvl: updated_vol}
                if resistance(history,row,n1,n2):
                    rr[history.index[row-1][1]] = history.high[row]
                    rounded_level = round(history.high[row],0)
                    if rounded_level not in list(rr_levels.keys()):
                        try:
                            rr_levels[rounded_level] = {history.high[row]: history.volume[row]}
                        except:
                            self.Debug(f"volume not available for {symbol} at {self.Time}")
                            rr_levels[rounded_level] = {history.high[row]: 0}
                    else:
                        previous_walvl = list(rr_levels[rounded_level].keys())[0]
                        previous_vol = list(rr_levels[rounded_level].values())[0]
                        new_lvl_input = history.high[row]
                        try:
                            new_lvl_vol = history.volume[row]
                        except:
                            new_lvl_vol = 0
                        if (previous_vol + new_lvl_vol) == 0:
                            updated_walvl = (previous_walvl + new_lvl_input) / 2
                        else:
                            updated_walvl = (previous_walvl * previous_vol + new_lvl_input * new_lvl_vol) / (previous_vol + new_lvl_vol)
                        updated_vol = previous_vol + new_lvl_vol
                        rr_levels[rounded_level] = {updated_walvl: updated_vol}
                    if rounded_level not in list(all_levels.keys()):
                        try:
                            all_levels[rounded_level] = {history.high[row]: history.volume[row]}
                        except:
                            all_levels[rounded_level] = {history.high[row]: 0}
                    else:
                        previous_walvl = list(all_levels[rounded_level].keys())[0]
                        previous_vol = list(all_levels[rounded_level].values())[0]
                        new_lvl_input = history.high[row]
                        try:
                            new_lvl_vol = history.volume[row]
                        except:
                            new_lvl_vol = 0
                        if (previous_vol + new_lvl_vol) == 0:
                            updated_walvl = (previous_walvl + new_lvl_input) / 2
                        else:
                            updated_walvl = (previous_walvl * previous_vol + new_lvl_input * new_lvl_vol) / (previous_vol + new_lvl_vol)
                        updated_vol = previous_vol + new_lvl_vol
                        all_levels[rounded_level] = {updated_walvl: updated_vol}
            self.support[symbol] = ss
            self.resistance[symbol] = rr
            self.support_levels[symbol] = ss_levels
            self.resistance_levels[symbol] = rr_levels
            self.all_levels[symbol] = all_levels
            
            # Support levels S1-S3 and their rvol
            try:
                PM_vol_total_2 = np.nansum(history['volume'].values)
            except:
                PM_vol_total_2 = 0
            if PM_vol_total_2 == 0:
                PM_vol_total_2 = 1
            sorted_s_l = sorted(self.support_levels[symbol].items(), key=lambda x: sum(x[1].values()), reverse=True)
            if len(sorted_s_l) >= 3:
                sorted_s_l_3 = sorted_s_l[:3]
            else:
                place_to_fill = 3 - len(sorted_s_l)
                sorted_s_l_3 = sorted_s_l
                for i in range(place_to_fill):
                    sorted_s_l_3 = sorted_s_l_3 + [(0.0,{0.0:0.0})] 
            sorted_support_levels = sorted(sorted_s_l_3, key=lambda x: x[0])
            pm_s3 = list(sorted_support_levels[0][1].keys())[0]
            pm_s3_rvol = list(sorted_support_levels[0][1].values())[0]/PM_vol_total_2
            pm_s2 = list(sorted_support_levels[1][1].keys())[0]
            pm_s2_rvol = list(sorted_support_levels[1][1].values())[0]/PM_vol_total_2
            pm_s1 = list(sorted_support_levels[2][1].keys())[0]
            pm_s1_rvol = list(sorted_support_levels[2][1].values())[0]/PM_vol_total_2
            # Resistance levels r1-r3 and their rvol 
            sorted_r_l = sorted(self.resistance_levels[symbol].items(), key=lambda x: sum(x[1].values()), reverse=True)
            if len(sorted_r_l) >= 3:
                sorted_r_l_3 = sorted_r_l[:3]
            else:
                place_to_fill = 3 - len(sorted_r_l)
                sorted_r_l_3 = sorted_r_l
                for i in range(place_to_fill):
                    sorted_r_l_3 = sorted_r_l_3 + [(0.0,{0.0:0.0})] 
            sorted_resistance_levels = sorted(sorted_r_l_3, key=lambda x: x[0])
            pm_r3 = list(sorted_resistance_levels[2][1].keys())[0]
            pm_r3_rvol = list(sorted_resistance_levels[2][1].values())[0]/PM_vol_total_2
            pm_r2 = list(sorted_resistance_levels[1][1].keys())[0]
            pm_r2_rvol = list(sorted_resistance_levels[1][1].values())[0]/PM_vol_total_2
            pm_r1 = list(sorted_resistance_levels[0][1].keys())[0]
            pm_r1_rvol = list(sorted_resistance_levels[0][1].values())[0]/PM_vol_total_2

            #Record pm_range
            self.pm_range[symbol] = pm_high - pm_low

            #self.Debug(f"support and resistance levels ready for {symbol} at {self.Time}")
            self.df_store.loc[len(self.df_store)] = [self.Time, symbol.Value, self.gap_combined[symbol], self.rvol_PM[symbol], pm_low, pm_s1, pm_s2, pm_s3, pm_s1_rvol, pm_s2_rvol, pm_s3_rvol, pm_high, pm_r1, pm_r2, pm_r3, pm_r1_rvol, pm_r2_rvol, pm_r3_rvol]
            self.df_store_day.loc[symbol] = [self.Time, symbol.Value, self.gap_combined[symbol], self.rvol_PM[symbol], pm_low, pm_s1, pm_s2, pm_s3, pm_s1_rvol, pm_s2_rvol, pm_s3_rvol, pm_high, pm_r1, pm_r2, pm_r3, pm_r1_rvol, pm_r2_rvol, pm_r3_rvol]
            #self.Debug(f"PM data successfully saved for {symbol} at {self.Time}")
        self.Debug(f"All PM data successfully collected at {self.Time}")
        #self.day_count +=1
        #self.ObjectStore.Save(f"{self.ProjectId}/PM_data_2022-1-1_{self.day_count} days out", self.df_store.reset_index().to_json(date_unit='ns'))
        self.Debug(f"object store as name {self.ProjectId}/PM_data_2022-1-1_{self.day_count} days out")
        previous_day_count = self.day_count - 1
        #self.ObjectStore.Delete(f"{self.ProjectId}/PM_data_2022-1-1_{previous_day_count} days out")
        self.Debug(f"{self.ProjectId}/PM_data_2022-1-1_{previous_day_count} days out deleted")

        """
        # Clear all dictionary to start fresh the next day
        self.gap.clear()
        self.gap_neg.clear()
        self.gap_combined.clear()
        self.rvol_PM.clear()
        self.support.clear()
        self.support_levels.clear() 
        self.resistance.clear()
        self.resistance_levels.clear()
        self.all_levels.clear()
        self.Debug(f"dictionaries cleared for the next day")
        """   
    def ClosePositions(self):
        self.Debug(f"Enter into ClosePositions function at {self.Time}")       
        if self.Portfolio.Invested:
            self.Liquidate()
            self.Debug(f"All stocks liquidated. Daily P&L at {self.daily_PL} at {self.Time}")
        openOrders = self.Transactions.GetOpenOrders()
        if len(openOrders)> 0:
            for x in openOrders:
                self.Transactions.CancelOrder(x.Id)
                self.Debug(f"Open order for {x} cancelled.")
        # empty the daily pm data tracking df
        self.df_store_day = self.df_store_day.drop(self.df_store_day.index)
        self.Debug(f"Cumulative number of winning trades are {self.winning_trades} with {self.winning_PL} cumulative winnings")
        self.Debug(f"Cumulative number of losing trades are {self.losing_trades} with {self.losing_PL} cumulative winnings")

def support(df1, l, n1, n2): #n1 n2 before and after candle l
    for i in range(l-n1+1, l+1):
        if(df1.low[i]>df1.low[i-1]):
            return 0
    for i in range(l+1,l+n2+1):
        if(df1.low[i]<df1.low[i-1]):
            return 0
    return 1

def resistance(df1, l, n1, n2): #n1 n2 before and after candle l
    for i in range(l-n1+1, l+1):
        if(df1.high[i]<df1.high[i-1]):
            return 0
    for i in range(l+1,l+n2+1):
        if(df1.high[i]>df1.high[i-1]):
            return 0
    return 1

# Read a key string to extract dataframe in the right format
def symbol_extractor(key, algorithm):
    Project_key = key
    df = pd.read_json(algorithm.ObjectStore.Read(Project_key))
    # Data Cleaning
    # Transform time column back to datetime format
    df['time'] = pd.to_datetime(df['time'])
    # Set time and symbol column as index
    df.set_index(['time', 'symbol'], inplace=True)
    # Remove index column
    df = df.drop('index', axis=1)
    dictionary = {}
    for (time, symbol), _ in df.iterrows():
        dictionary.setdefault(time, []).append(symbol)
    return dictionary    

# Updated version to deal wtih multi-day period. Original function cannot handle non-continuous datetime series
# To extract 10 seconds trading history for a symbol 15 mins from open (9:30-9:45)
def Consolidator_2(algorithm, symbol, start_time, end_time, base_freq = Resolution.Second, consol_freq = 10):
    Sym = algorithm.AddEquity(symbol) # symbol is a string HLGN
    history = algorithm.History(Sym.Symbol, start_time, end_time, base_freq, extendedMarketHours = True)
    history_reset = history.reset_index(level="symbol")
    grouped = history_reset.groupby(pd.Grouper(freq='D'))
    df_ls = []
    for date, group in grouped:
        # Store each day's DataFrame in the list
        df_ls.append(group.copy())
    df_consol_ls = []
    for df in df_ls:
        if base_freq == Resolution.Second:
            str_freq = f"{consol_freq}S"
        elif base_freq == Resolution.Minute:
            str_freq = f"{consol_freq}Min"
        try:
            df_his = df.resample(str_freq).apply(lambda x: pd.Series({
                'symbol': x['symbol'].iloc[0],
                'askclose': x['askclose'].iloc[-1],
                'askhigh' : x['askhigh'].max(),
                'asklow': x['asklow'].min(),
                'askopen': x['askopen'].iloc[0],
                'asksize': x['asksize'].sum(),
                'bidclose': x['bidclose'].iloc[-1],
                'bidhigh' : x['bidhigh'].max(),
                'bidlow': x['bidlow'].min(),
                'bidopen': x['bidopen'].iloc[0],
                'bidsize': x['bidsize'].sum(),
                'close': x['close'].iloc[-1],
                'high' : x['high'].max(),
                'low': x['low'].min(),
                'open': x['open'].iloc[0],
                'volume': x['volume'].sum()   
            }))
            df_consol_ls.append(df_his)
        except:
            print(f"Consolidator function: {symbol} has incomplete info from history")
            df_consol_ls.append(pd.DataFrame())
    ten_sec_his = pd.concat(df_consol_ls)
    ten_sec_his_reset = ten_sec_his.reset_index(level="time")
    ten_sec_his_reset.set_index(['symbol','time'], inplace=True)
    return ten_sec_his_reset

# find the slope for a multi-index series (such as pre_trading_history['close'])
def find_slope(multi_index_series):
    single_index_series = multi_index_series.droplevel(level=0)
    time_stamps = single_index_series.index.values
    x = pd.to_numeric(time_stamps)
    y = single_index_series.values
    m, _ = np.polyfit(x,y,1)
    return m

# DATA STRCUTRE TO STORE VWAP AND ATR
class SymbolData:
    def __init__(self,symbol,algo):
        self.algo = algo
        self.symbol = symbol
        #self.vwap = algo.VWAP(self.symbol)
        self.TenSecConsolidator = TradeBarConsolidator(timedelta(seconds=10))
    def Update(self,bar):
        self.vwap.Update(bar)