# region imports
from AlgorithmImports import *
from QuantConnect.DataSource import FamaFrench
# endregion

class IdiosyncraticReversionAlgorithm(QCAlgorithm):

    def initialize(self) -> None:
        self.set_start_date(2004, 1, 1)
        self.set_end_date(2024, 12, 31)
        # Large enough that the smallest value-weighted target (>$10k) is meaningful.
        self.set_cash(100000000)
        self.settings.seed_initial_prices = True

        # --- Fama-French five-factor panel ---
        self._ff_symbol = self.add_data(FamaFrench, 'FF', Resolution.DAILY).symbol
        self._ff_fields = ['market_excess_return', 'smb', 'hml', 'rmw', 'cma', 'risk_free_rate']
        self._ff_labels = ['Mkt-RF', 'SMB', 'HML', 'RMW', 'CMA', 'RF']

        # --- universe selection ---
        self._us_mics = {"XNYS", "XNAS", "XASE"}
        self._sic_financial_min = 6000
        self._sic_financial_max = 6999
        self._common_stock_type = "ST00000001"
        date_rule = self.date_rules.month_start("SPY")
        self.universe_settings.resolution = Resolution.DAILY
        self.universe_settings.schedule.on(date_rule)
        self._universe = self.add_universe(self._select_assets)

        # --- OLS regression + rebalance state ---
        self._mktcap = {}
        self._mic = {}
        self.settings.min_absolute_portfolio_target_percentage = 0.0
        self.settings.minimum_order_margin_portfolio_percentage = 0.0

        self.schedule.on(date_rule, self.time_rules.at(8, 0), self._rebalance)

    def _select_assets(self, fundamentals: List[Fundamental]) -> List[Symbol]:
        selected = []
        for f in fundamentals:
            sr = f.security_reference
            if sr.mic not in self._us_mics:
                continue
            if sr.security_type != self._common_stock_type:
                continue
            if not sr.is_primary_share:
                continue
            sic = f.asset_classification.sic
            if self._sic_financial_min <= sic <= self._sic_financial_max:
                continue
            book_equity = f.financial_statements.balance_sheet.stockholders_equity.value
            if np.isfinite(book_equity) and book_equity > 0:
                selected.append(f.symbol)
                self._mktcap[f.symbol] = float(f.market_cap)
                self._mic[f.symbol] = sr.mic
        return selected

    def _ff_monthly_panel(self, start: datetime, end: datetime) -> "pd.DataFrame":
        # Monthly factor panel (dividend-inclusive, compounded from daily) by period.
        bars = self.history[FamaFrench](self._ff_symbol, start, end, Resolution.DAILY)
        rows = {}
        accord = None
        cur_ym = None
        for b in bars:
            ym = pd.Period(b.end_time, freq='M')
            if cur_ym is None:
                cur_ym = ym
                accord = {f: 1.0 for f in self._ff_fields}
            elif ym != cur_ym:
                rows[cur_ym] = {lbl: accord[f] - 1.0 for f, lbl in zip(self._ff_fields, self._ff_labels)}
                cur_ym = ym
                accord = {f: 1.0 for f in self._ff_fields}
            for f in self._ff_fields:
                accord[f] *= (1.0 + float(getattr(b, f)))
        if accord is not None:
            rows[cur_ym] = {lbl: accord[f] - 1.0 for f, lbl in zip(self._ff_fields, self._ff_labels)}
        return pd.DataFrame.from_dict(rows, orient='index')

    def _rebalance(self) -> None:
        now = self.Time.replace(hour=0, minute=0, second=0, microsecond=0)
        p = pd.Period(now, 'M')  # formation month t (first trading day)

        # Window months t-60..t-1.
        start_month = (p - 60).start_time
        end_month = (p - 1).end_time

        ff = self._ff_monthly_panel(start_month, end_month)
        ff = ff.sort_index()
        t_minus = ff.index[-1]  # previous month t-1
        prev_factors = ff.loc[t_minus]
        prev = {lbl: float(prev_factors[lbl]) for lbl in self._ff_labels}

        panel = list(self._universe.selected)

        hist_start = (p - 61).start_time
        hist = self.history(panel, hist_start, p.start_time, Resolution.DAILY)
        closes_by_symbol = {}
        if hist is not None and not hist.empty:
            df = hist.reset_index()
            df['ym'] = df['time'].dt.to_period('M')
            for sym, g in df.groupby('symbol'):
                closes_by_symbol[sym] = g.groupby('ym')['close'].last()

        factor_df = ff

        idio = {}
        for sym, closes in closes_by_symbol.items():
            monthly_ret = closes.pct_change().dropna()
            stock_ret = monthly_ret.reindex(factor_df.index).dropna()
            if len(stock_ret) < 36:
                continue
            if stock_ret.index[-1] != t_minus:
                continue
            fmat = factor_df.loc[stock_ret.index]
            y = stock_ret.values - fmat['RF'].values
            X = np.column_stack([
                fmat['Mkt-RF'].values, fmat['SMB'].values, fmat['HML'].values,
                fmat['RMW'].values, fmat['CMA'].values,
            ])
            Xd = np.column_stack([np.ones(len(X)), X])
            beta, _, _, _ = np.linalg.lstsq(Xd, y, rcond=None)
            betas = beta[1:6]
            r_excess = float(stock_ret.iloc[-1]) - prev['RF']
            systematic = sum(bt * prev[lbl] for bt, lbl in zip(betas, ['Mkt-RF', 'SMB', 'HML', 'RMW', 'CMA']))
            idio[sym] = r_excess - systematic

        ranked = sorted(idio.items(), key=lambda kv: kv[1])

        # --- quintile formation & value-weighted legs ---
        nyse_names = [s for s, _ in ranked if s in self._mic and self._mic[s] == "XNYS"]
        nyse_idio = [idio[s] for s in nyse_names]
        n = len(nyse_idio)
        sorted_nyse = sorted(nyse_idio)
        cuts = [sorted_nyse[int(round(n * (i + 1) / 5)) - 1] for i in range(4)]

        quints_low = []
        quints_high = []
        for s, v in ranked:
            cut_idx = 0
            for c in cuts:
                if v > c:
                    cut_idx += 1
            if cut_idx == 0:
                quints_low.append(s)
            elif cut_idx == 4:
                quints_high.append(s)

        def leg_weights(names: List[Symbol], sign: float) -> dict:
            caps = {s: self._mktcap.get(s, np.nan) for s in names}
            caps = {s: c for s, c in caps.items() if np.isfinite(c) and c > 0}
            tot = sum(caps.values())
            return {s: sign * 0.5 * (c / tot) for s, c in caps.items()}

        long_weights = leg_weights(quints_low, +1)
        short_weights = leg_weights(quints_high, -1)

        # --- submit orders (fill at this trading day's open) ---
        targets = []
        for sym, w in long_weights.items():
            targets.append(PortfolioTarget(sym, w))
        for sym, w in short_weights.items():
            targets.append(PortfolioTarget(sym, w))

        self.set_holdings(targets, liquidate_existing_holdings=True)

# region imports
from AlgorithmImports import *
# endregion

SPEC = "# Idiosyncratic Reversion — Short-Term Reversal in the Firm-Specific Return Component\n\n**Deutsche Bank \"Academic Insights\" (AI October 2025), item #99 — \"Cash equities — Reversion #2: Idiosyncratic Reversion\" (p.297).**\n**Underlying paper:** Graef, Hoechle & Schmid, *\"Firm-specific versus systematic momentum\"* (December 2024), SSRN 5053270. Sample CRSP/Compustat July 1963 – December 2019; factor returns from the Ken French library. No public author code.\n\nImplement the **full method** below: the rolling five-factor decomposition of each stock's monthly return into a systematic and a firm-specific component, and the monthly value-weighted quintile reversal portfolio sorted on the *firm-specific* component of last month's return. Do not build a reduced version — in particular, do not substitute the raw last-month return for the firm-specific component: the paper shows the raw-return reversal is less than half as strong, precisely because it is contaminated by the systematic part.\n\n---\n\n## Structured brief (for the QC Assistants)\n\n**Concept.** Decompose each stock's monthly excess return into a systematic component (what its current Fama–French five-factor loadings times that month's factor returns explain) and an idiosyncratic component (the remainder). Each month, rank all eligible stocks by the idiosyncratic component of the *previous month's* return, form value-weighted quintiles with NYSE breakpoints, and hold the lowest-quintile stocks LONG and the highest-quintile stocks SHORT for one month — a short-term reversal book operating purely on firm-specific price moves.\n\n**Why the pattern exists.** Short-term reversal is compensation for liquidity provision: a stock's outsized one-month move on firm-specific news overshoots as demanders of immediacy push price beyond fundamentals, and the correction accrues to whoever takes the other side. Sorting on the *raw* last-month return dilutes this effect, because the systematic part of a month's return does not revert — it mildly *continues* (one-month factor momentum) — so raw-return reversal mixes a reverting firm-specific signal with a continuing systematic one. Stripping out the factor-driven component isolates the overreaction, and the paper's measured reversal on the firm-specific component is more than twice the raw-return reversal, monotone across quintiles, and robust across factor models and beta-estimation windows.\n\n**Investment universe.** US ordinary common stocks listed on NYSE, AMEX, or NASDAQ, excluding financial firms and firms with negative book equity (the paper adopts these exclusions from Hou, Xue & Zhang 2020; financials are operationalized per that convention as SIC 6000–6999). One listing per company (primary share class — the platform realization of a CRSP common-share sample). A stock must additionally have enough return history to estimate its factor loadings (at least 36 of the past 60 months).\n\n**Signal.** The idiosyncratic component of last month's return: last month's excess return minus the product of the stock's current five-factor loadings and last month's factor returns.\n\n**Portfolio construction & rebalance.** Monthly. Quintile breakpoints computed from NYSE-listed stocks only; within quintiles, weights proportional to prior-month-end market equity. LONG the lowest-idiosyncratic-return quintile, SHORT the highest, legs scaled to +0.5 and −0.5 of portfolio value (gross exposure 1x, dollar-neutral). Decisions are made on the first trading day of the month before the market opens (8AM Eastern), from information through the prior month-end close; orders fill at that day's open. Hold until the next monthly rebalance.\n\n**Reference.** Graef, Frank, Daniel Hoechle & Markus Schmid, *Firm-specific versus systematic momentum*, SSRN 5053270 — §3 (sample construction), §4.2 and Equations (2)–(3) (decomposition and sorting), Table 2 (the short-term idiosyncratic-return sort). Factor definitions: the Fama–French five research factors and the one-month risk-free rate of the Ken French Data Library.\n\n---\n\n## Implementation specifics from the source paper\n\n### 1. Universe and data\n\n- NYSE, AMEX, or NASDAQ listing; ordinary common stock (one listing per company — primary share class).\n- Exclude financial firms and firms whose most recent book equity is negative — the paper's exclusions, adopted from Hou, Xue & Zhang (2020); financials per that convention are SIC 6000–6999. Accounting data is usable only once publicly filed.\n- Monthly total returns including dividends; excess returns subtract the one-month risk-free rate. Delistings are handled by the platform's delisting processing.\n- Factor inputs: the published monthly Fama–French five research-factor returns (market-minus-bills, size, value, profitability, investment) and the risk-free series.\n\n### 2. Rolling factor loadings\n\nFor each stock at each month-end t, estimate the five factor loadings by an ordinary least-squares regression (with intercept) of the stock's monthly excess returns on the five factor returns over the window **months t−60 through t−1**, requiring **at least 36 non-missing monthly observations** within the window. Stocks failing the requirement are ineligible that month.\n\n### 3. Return decomposition (the signal)\n\nUsing the loadings estimated at month t, the systematic part of last month's return and the signal are:\n\n$$\n\\hat{r}_{i,t-1} = \\sum_{f=1}^{5} \\hat{\\beta}^{f}_{i,t}\\, r^{f}_{t-1}, \\qquad\n\\hat{\\varepsilon}_{i,t-1} = r^{e}_{i,t-1} - \\hat{r}_{i,t-1}\n$$\n\nwhere `r^f` are the monthly factor returns and `r^e` is the stock's month t−1 return in excess of the risk-free rate. The regression intercept is not part of the systematic return — only the factor-loading terms are.\n\n### 4. Portfolio formation\n\nAt each month-end t:\n\n1. Rank eligible stocks by the signal ε̂ of item 3 and cut into quintiles using breakpoints computed from **NYSE-listed stocks only**.\n2. Within each quintile, weight stocks by their month-end t market equity (value-weighted).\n3. Hold the **lowest** quintile LONG and the **highest** quintile SHORT for the following month (the reversal direction: last month's firm-specific losers are bought, winners sold). Legs scale to +0.5 / −0.5 of portfolio value — gross 1x, net 0.\n4. The decision runs on the first trading day of month t+1 before the open (8AM Eastern); orders fill at that day's open; positions are held unchanged until the next monthly rebalance. A stock that delists mid-month leaves the book at its delisting.\n\n### 5. Validation period\n\nThe final full-period validation backtest runs **January 1, 2004 through December 31, 2024**. (The paper's sample begins in 1963 with strategies evaluated from 1966; the platform's US equity history begins in 1998, and the 60-month beta window with its 36-month minimum makes 2004 the earliest start at full strength.)\n\n---\n"