Coarse/Fine Universe Selection with Customer SymbolData Inidcators for Mometum - Please register to receive data for symbol using the AddSecurity() function. in Indicators.cs:line 2187

#region imports
from AlgorithmImports import *
#endregion

import numpy as np
import pandas as pd
from scipy.stats import linregress
from collections import deque
from datetime import timedelta
from datetime import datetime
import math
class MomentumRanker(PythonIndicator):
    
    def momentum(self, closes):
        returns = np.log(closes)
        x = np.arange(len(returns))
        slope, _, rvalue, _, _ = linregress(x, returns)
        return ((1 + slope) ** 252) * (rvalue ** 2)  # annualize slope and multiply by R^2

    def __init__(self, period):
        self.Time = datetime.min
        self.period = period
        self.closes = deque(maxlen=period)  # queue for storing closes
        self.Value = 0
    
    def __repr__(self):
        return f"Momentum -> IsReady: {self.IsReady}. Time: {self.Time}. Value: {self.Value}." 

    def IsReady(self):
        return (self.Value > 0) # return(len(self.closes)==self.closes.maxlen)
    
    def Update(self, bar:TradeBar):
        self.closes.append(bar.Close)
        self.Time = bar.Time
    
        self.Value = 0

        if len(self.closes) == self.closes.maxlen:
            self.Value = self.momentum(np.array([*self.closes])) # convert deque to list then to np.array
        
        return (self.Value > 0)

class SymbolData:
    def __init__(self, algorithm, symbol, filter_period):
        self.algorithm = algorithm
        self.symbol = symbol

        self.filter_period = filter_period
        self.volatility_period = 20
        self.momentum_period = 90 

        self.momentum = MomentumRanker(self.momentum_period)
        self.atr = self.algorithm.ATR(symbol, self.volatility_period)
        self.filter = self.algorithm.SMA(symbol, self.filter_period)

        algorithm.RegisterIndicator(self.symbol, self.momentum)
        algorithm.RegisterIndicator(self.symbol, self.filter)
        algorithm.RegisterIndicator(self.symbol, self.atr)
        
        algorithm.WarmUpIndicator(self.symbol, self.atr)
        algorithm.WarmUpIndicator(self.symbol, self.filter)

        # warmup momentum indicators
        trade_bars = algorithm.History[TradeBar](symbol, self.momentum_period, Resolution.Daily)
        for bar in trade_bars:
            self.momentum.Update(bar)
    
    def UpdateIndicators(self, data:TradeBar): 
        # update the indicators
        self.momentum.Update(data)
        self.filter.Update(data.Time, data.Close)  # SMA takes time, IndicatorDataPoint, may need to construct and IndicatorDataPoint
        self.atr.Update(data)                   

    def IsReady(self):
        return self.momentum.IsReady and self.atr.IsReady and self.filter.IsReady

   

class MomentumEffectAlgorithm(QCAlgorithm):

    def Initialize(self):
        self.SetStartDate(2016, 1, 1)  # Set Start Date
        self.SetEndDate(2017, 1, 1)    # Set End Date       
        self.SetCash(100000)           # Set Strategy Cash

        self.UniverseSettings.Resolution = Resolution.Daily
        self.UniverseSettings.DataNormalizationMode = DataNormalizationMode.Adjusted
        self.UniverseSettings.MinimumTimeInUniverse = timedelta(days=1)

        # self.ActiveSecurities dictionary of all the securities currently in universe
        self.symbol_dictionary = {} # dictionary for holding SymbolData key by symbol
        self.excluded_securities = ['SPY']

        # adjust these in line with Clenow's approach - want approx 20 stocks in the portfolio
        self.num_coarse = 500 # Number of symbols selected at Coarse Selection
        self.num_fine = 200    # Number of symbols selected at Fine Selection
        self.num_positions = 20     # Number of symbols with open positions
        self.risk_factor = 0.001    # targeting 10 basis point move per day
        self.momentum_threshold = 0.0  # TODO: check the value to see if this is reasonable
        
        self.index_filter_period = 200

        # variables to control the portfolio rebalance and the stock selection reranking
        self.month = -1
        self.dayofweek = 3  # day of week to rerank
        self.rebalance = False
        self.rerank = False

        # create the equities universe
        self.AddUniverse(self.CoarseSelectionFunction, self.FineSelectionFunction)

        # this is causing serious problems. I don't want to trade SPY, just to use it and its 
        # 200 sma as a filter on the  market, but it ends up in Portfolio holdings and screwing up the data feed 
        # via the factor file adjustments

        self.market = self.AddEquity("SPY", Resolution.Daily).Symbol

        # self.SetBenchmark(self.market)

        # set Brokerage model and Fee Structure
        self.SetBrokerageModel(BrokerageName.InteractiveBrokersBrokerage, AccountType.Margin)

        # set Free Cash to 2.5%
        self.Settings.FreePortfolioValuePercentage = 0.025
        
        """
        # can't use scheduling with coarse / fine universe selection
        # but TODO: can probably schedule the rerank and rebalance methods, to run after OnData
        self.Schedule.On(self.DateRules.MonthEnd(self.index, daysOffset = 0),
                         self.TimeRules.BeforeMarketClose("SPY", 60),
                         self.RerankMomentum)
        
        self.Schedule.On(self.DateRules.Every(DayOfWeek.Wednesday),
                         self.TimeRules.BeforeMarketClose("SPY", 120),
                         self.RebalancePortfolio)
        
        # Selection will run on every week on Monday 2 hours before market close
        self.AddUniverseSelection(ScheduledUniverseSelectionModel(
                                self.DateRules.Every(DayOfWeek.Monday),
                                self.TimeRules.AfterMarketClose("SPY", 15),
                                self.SelectSymbols))
        
        """

    # my methods

    def CalculateRiskParityPositionSize(self, symbol, atr)-> float:
        amountToRisk = self.Portfolio.TotalPortfolioValue * self.risk_factor
        quantity = self.CalculateOrderQuantity(symbol, 0.01) # get the quanity of shares for holding of 1%
        shares = amountToRisk / atr
        rounded_holding = 0.0
        if quantity != 0:
            holding_percent = (shares/quantity)/100
            rounded_holding = round(holding_percent, 3) # round down to 0.001
        return rounded_holding

    # QC methods
    def OnWarmUpFinished(self) -> None:
        self.Log("Equities Momentum Algorithm Ready")

    
    # this article explains when universe selection is done (12:00am each day) and the order of the calls
    # coarse, fine, onsecuritieschanged, then ondata
    # https://www.quantconnect.com/forum/discussion/6485/onsecuritieschanged-questions/p1

    # may eventually switch this out for small or mid cap stocks
    def CoarseSelectionFunction(self, coarse):
        '''Drop securities which have no fundamental data or have too low prices. Select those with highest by dollar volume'''
        
        if not self.Time.weekday() == 1: # monday
            return Universe.Unchanged

        self.rerank = True

        selected = sorted([x for x in coarse if x.HasFundamentalData and x.Price > 5], key=lambda x: x.DollarVolume, reverse=True)
        return [x.Symbol for x in selected[:self.num_coarse]]

    # may not really want to do anything here - might be that coarse filtering is enough
    def FineSelectionFunction(self, fine):
        '''Select security with highest market cap, and common stock'''
        filtered = sorted(fine, key=lambda f: f.MarketCap and 
                                              f.SecurityReference.SecurityType == 'ST00000001', # stock is common stock, ST00000002 is preferred stock, 
                                              reverse=True)

        return [x.Symbol for x in filtered[:self.num_fine]]      


    def OnData(self, data):

        if data.Bars.ContainsKey(self.market) and self.market not in self.symbol_dictionary.keys():
            self.symbol_dictionary[self.market] = SymbolData(self, self.market, 200)
        
        if (self.dayofweek != self.Time.weekday()):
            self.rerank = True
            
        else:
            return

        if (self.month != self.Time.month):
            self.rebalance = True
            self.month = self.Time.month

        self.Log(f'OnData: Update Indicators in Symbol Dictionary')
        for symbol in self.Securities.keys():
            if symbol in self.symbol_dictionary.keys():
                if data.Bars.ContainsKey(symbol):
                    self.symbol_dictionary[symbol].UpdateIndicators(data[symbol])
                else:
                    self.Debug(f'{symbol.ID} in Symbol Dictionary but missing Slice Data')
            else:
                self.symbol_dictionary[symbol] = SymbolData(self, symbol, 100)

        # rerank weekly, rebalance monthly
        if not self.rerank:
            return

        liquidated = []
        purchased = []

        self.Log('OnData: Momentum Reranking')

        # rerank securities based on momentum rank above threshold and close above sma filter
        mom_list = [key for key in self.symbol_dictionary.keys() if self.symbol_dictionary[key].momentum.IsReady]
        mom_threshold = [key for key in mom_list if self.symbol_dictionary[key].momentum.Current.Value >= self.momentum_threshold]
        sorted_mom = sorted(mom_threshold, key=lambda x: self.symbol_dictionary[x].momentum.Current.Value, reverse=True)

        # filter selected to chose only those with a price above their sma 100 
        for symbol in sorted_mom:
            if self.symbol_dictionary[symbol].filter.IsReady:
                if self.CurrentSlice[symbol].Close < self.symbol_dictionary[symbol].filter.Current.Value:
                    sorted_mom.remove(symbol)
        
        selected = sorted_mom[:self.num_positions] 
        
        momentum_selection = []
        for symbol in selected:
            momentum_selection.append(symbol.ID.ToString().split(' ')[0])
        self.Log(f'Selected by Momentum: {momentum_selection}')

        # Sell
        # Liquidate securities that are in our portfolio, but no longer in the top momentum rankings
        # and have closed below the filter
        for symbol, holding in self.Portfolio.items():
            if (symbol not in selected) and (holding.Quantity > 0):
                close = data[symbol].Close
                filterVal = self.symbol_dictionary[symbol].filter.Current.Value
                if close < filterVal:
                    self.Liquidate(symbol, f'{str(symbol)} as closed {close} is below filter {filterVal}')
                    liquidated.append(symbol.ID.ToString().split(' ')[0])
                      
        # log which ones were removed and current margin remaining now
        self.Log(f'OnData: {liquidated} liquidated. Current Margin: {self.Portfolio.MarginRemaining}')

        # Buy those we don't already have
        buying = (data[self.market].Close > self.symbol_dictionary[self.market].filter.Current.Value)
        self.Log(f'OnData: Buying = {buying} [{data[self.market].Close}, {self.symbol_dictionary[self.market].filter.Current.Value}]')

        # if we pass our index filter, buy selected securities (check not already in holdings first though)
        if buying:
            buffer = (self.Portfolio.TotalPortfolioValue / 100) # 1% buffer
            for symbol in selected:
                if (not symbol in self.Portfolio.keys()) and (not symbol in self.excluded_securities):
                    remaining_margin = self.Portfolio.MarginRemaining
                    if remaining_margin > buffer:
                        holding = self.CalculateRiskParityPositionSize(symbol, self.volatility[symbol].Current.Value)
                        self.SetHoldings(symbol, holding)
                        purchased.append(f"{symbol.ID.ToString().split(' ')[0]} {holding:.2f}")
            # log which ones were purchased
            self.Log(f'OnData: {purchased} purchased based on momentum rankings')
        
        # check current holdings
        holdings = []
        for symbol, holding in self.Portfolio.items(): 
            if holding.Quantity > 0:
                holdings.append(f"{symbol.ID.ToString().split(' ')[0]} {holding.Quantity:.2f}")
        self.Log(f'Portfolio Holdings: {holdings}')

        portfolio_update = f'Total: {self.Portfolio.TotalPortfolioValue}, Cash: {self.Portfolio.Cash}, Margin Used: {self.Portfolio.TotalMarginUsed}, Margin Remaining {self.Portfolio.MarginRemaining}'
        self.Log(f'Portfolio Value: {portfolio_update}')

        if not self.rebalance:
            return

        rebalanced = []

        for symbol, security_holding in self.Portfolio.items():
            # quantitylog = f'{str(symbol)}, qty = {security_holding.Quantity}'
            symbol = security_holding.Symbol
            if symbol in self.symbol_dictionary.keys():
                holding = self.CalculateRiskParityPositionSize(symbol, self.symbol_dictionary[symbol].atr.Current.Value)
                self.SetHoldings(symbol, holding)
                rebalanced.append(f"{symbol.ID.ToString().split(' ')[0]} {holding},")
            else:
               self.Debug(f'Error {str(symbol)} in portfolio holdings, but not in symbol_dictionary') 

        self.Log(f'Portfolio Weights: {rebalanced}')

        self.rebalance = False
        self.rerank = False

 
    def OnSecuritiesChanged(self, changes):

        # shifted the changes from OnSecuritiesChanged to be processed here via self.changes
        addedSymbols = []
        removedSymbols = []

        # Clean up securities list and indicator data for removed securities
        for security in changes.RemovedSecurities:
            symbol = security.Symbol
            self.Liquidate(symbol, f'{self.Time} {str(symbol)} no longer in Universe')  
            if symbol in self.symbol_dictionary.keys():
                self.symbol_dictionary.pop(symbol) 
                removedSymbols.append(symbol.ID.ToString().split(' ')[0])

        # Create indicators and warm them up for securities newly added to the universe
        for security in changes.AddedSecurities:
            symbol = security.Symbol
            if symbol not in self.symbol_dictionary.keys(): 
                self.symbol_dictionary[symbol] = SymbolData(self, symbol, 100)
                addedSymbols.append(symbol.ID.ToString().split(' ')[0])

        self.Log(f'SecuritiesChanged: Added: {addedSymbols}')
        self.Log(f'SecuritiesChanged: Removed: {removedSymbols}')

Added the code as I'm unable to attach the backtest (because of the Runtime error? Hopefully that's not the case).

Would appreciate some insight from the moderaters or someone experience enough to solve this issue. I've been trying to implement a strategy that uses coarse/fine filter universe selection an then applies customer indicator calculations to further rank the securities selection under both classic and algorithmic framework style strategies, using the SymbolData pattern and it keeps throwing issues no matter what I try. (And I've tried a lot of different approaches - mainly shifting responsibilities around amongst where the event methods are called in the algorithm engine sequence).