DolphinDB/modules/indicatorCalculator.dos

module fundit::indicatorCalculator

use fundit::dataPuller
use fundit::returnCalculator
use fundit::navCalculator
/*
 *  Annulized multiple
 */
def get_annulization_multiple(freq) {

  ret = 1;
  
  if (freq == 'd') {
    ret = 252; // We have differences here between Java and DolphinDB, Java uses 365.25 days
  } else if (freq == 'w') {
    ret = 52;
  } else if (freq == 'm') {
    ret = 12;
  } else if (freq == 'q') {
    ret = 4;
  } else if (freq == 's') {
    ret = 2;
  } else if (freq == 'a') {
    ret = 1;
  }
  
  return ret;
}


/*
 *     Trailing Return, Standard Deviation, Skewness, Kurtosis, Max Drawdown, VaR, CVaR
 *     @param ret: 收益表，需要有 entity_id, price_dat, end_date, nav 
 *     @param freq: 数据频率，d, w, m, q, s, a
 *     
 *     NOTE: standard deviation of Java version is noncompliant-GIPS annulized number
 *     
 *     Create:  20240904                                                                     Joey
 *                       TODO: var and cvar are silightly off compared with Java version
 *     
 */
def cal_basic_performance(ret, freq) {

    t = SELECT entity_id, max(end_date) AS end_date, max(price_date) AS price_date, min(price_date) AS min_date,
               //(nav.last() \ nav.first() - 1).round(6) AS trailing_ret,
               ((1+ret).prod()-1).round(6) AS trailing_ret,
               iif(price_date.max().month()-price_date.min().month()>12,
                   //(nav.last() \ nav.first()).pow(365 \(max(price_date) - min(price_date)))-1, 
                   //(nav.last() \ nav.first() - 1)).round(6) AS trailing_ret_a,
                   ((1+ret).prod()-1) * sqrt(get_annulization_multiple(freq)),
                   ((1+ret).prod()-1)).round(6) AS trailing_ret_a,
               ret.std() AS std_dev,
               ret.skew(false) AS skewness,
               ret.kurtosis(false) - 3 AS kurtosis,
               ret.min() AS wrst_month,
               max( 1 - nav \ nav.cummax() ) AS drawdown
        FROM ret
        GROUP BY entity_id;

    // var & cvar require return NOT NULL
    // NOTE: DolphinDB supports 4 different ways: normal, logNormal, historical, monteCarlo. we use historical
    t1 = SELECT entity_id, max(end_date) AS end_date, max(price_date) AS price_date,
                ret.VaR('historical', 0.95) AS var,
                ret.CVaR('historical', 0.95) AS cvar
         FROM ret
         WHERE ret.ret > - 1
         GROUP BY entity_id;

   return (SELECT * FROM t LEFT JOIN t1 ON t.entity_id = t1.entity_id AND t.end_date = t1.end_date AND t.price_date = t1.price_date);

}


/*
 *   Lower Partial Moment
 *   NOTE: risk free rate is used as Minimal Accepted Rate (MAR) here
 * 
 */
def cal_LPM(ret, risk_free_rate) {
    
    t = SELECT *, count(entity_id) AS cnt FROM ret WHERE ret > -1 CONTEXT BY entity_id;

    lpm = SELECT t.entity_id, max(t.end_date) AS end_date,
                 (sum (rfr.ret - t.ret) \ (t.cnt[0])).pow(1\1) AS lpm1, 
                 (sum2(rfr.ret - t.ret) \ (t.cnt[0])).pow(1\2) AS lpm2, 
                 (sum3(rfr.ret - t.ret) \ (t.cnt[0])).pow(1\3) AS lpm3
          FROM t 
          INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
          WHERE t.ret < rfr.ret
          GROUP BY t.entity_id;

    return lpm;
}

/*
 *     Downside Devision, Omega Ratio, Sortino Ratio, Kappa Ratio
 * 
 *     TODO: Java version of Downside Deviation (LPM2) uses cnt-1 as denominator to calculate mean excess return, which might be wrong
 *           Java version of Omega could be wrong because Java uses annualized returns and cnt-1 
 *           Java'version of Kappa could be very wrong
 *           
 */
def cal_omega_sortino_kappa(ret, risk_free_rate) {

    lpm = cal_LPM(ret, risk_free_rate);

    tb = SELECT t.entity_id, 
                l.lpm2[0] AS ds_dev,
                (t.ret - rfr.ret ).mean() \ l.lpm1[0] + 1 AS omega,
                (t.ret - rfr.ret ).mean() \ l.lpm2[0] AS sortino,
                (t.ret - rfr.ret ).mean() \ l.lpm3[0] AS kappa
              FROM ret t
              INNER JOIN lpm l ON t.entity_id = l.entity_id
              INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
              GROUP BY t.entity_id;

    return tb;
}


/*
 *   Alpha & Beta
 *   NOTE: alpha of Java version is noncompliant-GIPS annulized number
 */
def cal_alpha_beta(ret, bmk_ret, risk_free) {

    t = SELECT t.entity_id, t.end_date, t.ret, bmk.ret AS ret_bmk
        FROM ret t
        INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
        WHERE t.ret > -1
          AND bmk.ret > -1;

    beta = SELECT ret.beta(ret_bmk) AS beta FROM t GROUP BY entity_id;

    alpha = SELECT t.entity_id, (t.ret - rfr.ret).mean() - beta.beta[0] * (t.ret_bmk - rfr.ret).mean() AS alpha
            FROM t 
            INNER JOIN beta beta ON t.entity_id = beta.entity_id
            INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
            GROUP BY t.entity_id;

    return ( SELECT * FROM beta AS b INNER JOIN alpha AS a ON a.entity_id = b.entity_id  );
}

/*
 *    Winning Ratio, Tracking Error, Information Ratio
 *    TODO: Information Ratio is way off!
 *          Not sure how to describe a giant number("inf"), for now 999 is used
 */
def cal_benchmark_tracking(ret, bmk_ret) {

     t0 = SELECT t.entity_id, t.end_date, t.ret, bmk.ret AS ret_bmk, count(entity_id) AS cnt, (t.ret - bmk.ret) AS exc_ret
          FROM ret t
          INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
          WHERE t.ret > -1
            AND bmk.ret > -1
          CONTEXT BY t.entity_id;

     t = SELECT entity_id, 
                exc_ret.bucketCount(0:999, 1) \ cnt[0] AS winrate,
                exc_ret.std() AS track_error, 
                exc_ret.mean() / exc_ret.std() AS info
         FROM t0 GROUP BY entity_id

     return t;
}

/*
 *    Sharpe Ratio
 *    NOTE: Java version is noncompliant-GIPS annulized number
 */
def cal_sharpe(ret, std_dev, risk_free_rate) {

    sharpe = SELECT t.entity_id, (t.ret - rfr.ret).mean() / std.std_dev[0] AS sharpe
             FROM ret t
             INNER JOIN std_dev std ON t.entity_id = std.entity_id
             INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
             WHERE std.std_dev[0] <> 0
             GROUP BY t.entity_id;

    return sharpe;
}

/*
 *    Treynor Ratio
 */
def cal_treynor(ret, risk_free_rate, beta) {

    t = SELECT *, count(entity_id) AS cnt 
       FROM ret t 
       INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date 
       WHERE t.ret > -1
         AND rfr.ret > -1
       CONTEXT BY t.entity_id;
       
    treynor = SELECT t.entity_id, ((1 + t.ret).prod().pow(12\iif(t.cnt[0]<12, 12, t.cnt[0])) - (1 + t.rfr_ret).prod().pow(12\iif(t.cnt[0]<12, 12, t.cnt[0]))) / beta.beta[0] AS treynor
              FROM t
              INNER JOIN beta AS beta ON t.entity_id = beta.entity_id
              GROUP BY t.entity_id;

    return treynor;
}

/*
 *    Jensen's Alpha
 *    TODO: the result is slightly off
 */
def cal_jensen(ret, bmk_ret, risk_free_rate, beta) {

    jensen = SELECT t.entity_id, t.ret.mean() - rfr.ret.mean() - beta.beta[0] * (bmk.ret.mean() - rfr.ret.mean()) AS jensen
             FROM ret t
             INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
             INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
             INNER JOIN beta beta ON t.entity_id = beta.entity_id
             GROUP BY t.entity_id;
               
    return jensen;
}

/*
 *    Calmar Ratio
 *    TODO: the result is off
 *
 */
def cal_calmar(ret_a){

    calmar = SELECT entity_id, trailing_ret_a \ drawdown AS calmar
             FROM ret_a;

    return calmar;
}

/*
 *     Modigliani Modigliani Measure (M2)
 *     NOTE: M2 = sharpe * std(benchmark) + risk_free_rate
 *     NOTE: Java version is noncompliant-GIPS annulized number 
 */
def cal_m2(ret, bmk_ret, risk_free_rate) {

    m2 = SELECT t.entity_id, (t.ret - rfr.ret).mean() / t.ret.std() * bmk.ret.std() + rfr.ret.mean() AS m2
         FROM ret t
         INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
         INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
         GROUP BY t.entity_id;

    return m2;
}

/*
 *    Morningstar Return, Morningstar Risk-Adjusted Return
 *    
 *    TODO: Tax and loads are NOT taken care of
 *    TODO: Assume Chinese methodology using 3, 5, 10 as number of traling years
 *    
 *    NOTE: Morningstar methodology requires monthly return for calculation, so that "12" is hard-coded here
 * 
 * 
 */
def cal_ms_return(ret, risk_free_rate) {

    r = SELECT t.entity_id, t.end_date.max() AS end_date, t.price_date.max() AS price_date, t.price_date.min() AS min_date,
               ((1 + t.ret)\(1 + rfr.ret)).prod().pow(12\(t.end_date.max() - t.end_date.min()))-1 AS ms_ret_a,
               (1 + t.ret).pow(-2).mean().pow(-12/2)-1 AS ms_rar_a
        FROM ret t
        INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
        GROUP BY t.entity_id;

    return r;
}


/*
 *   Monthly Since_inception_date Indicator Calculation
 *   @param: ret <TABLE>: 收益表，NEED COLUMNS entity_id, price_dat, end_date, nav 
 *   @param index_ret <TABLE>: historical benchmark return table, NEED COLUMNS fund_id, end_date, ret
 *   @param risk_free <TABLE>: historical risk free rate table, NEED COLUMNS fund_id, end_date, ret
 *   @param freq <CHAR>: 数据频率，d, w, m, q, s, a
 *   
 *   @return: indicators table
 *   
 *   
 *   Create  20240904  模仿Java & python代码在Dolphin中实现，具体计算逻辑可能会有不同                          Joey
 *                     TODO: some datapoints require more data, we need a way to disable calculation for them
 *
 */
def cal_indicators(mutable ret, index_ret, risk_free, freq) {

    if (! freq IN ['d', 'w', 'm', 'q', 's', 'a']) return null;

    // sorting for correct first() and last() value
    ret.sortBy!(['entity_id', 'price_date'], [1, 1]);

    // 收益、标准差、偏度、峰度、最大回撤、VaR, CVaR
    rtn = cal_basic_performance(ret, freq);

    // alpha, beta
    alpha_beta = cal_alpha_beta(ret, index_ret, risk_free);

    // 胜率、跟踪误差、信息比率
    bmk_tracking = cal_benchmark_tracking(ret, index_ret);

    // 夏普
    sharpe = cal_sharpe(ret, rtn, risk_free);

    // 特雷诺
    treynor = cal_treynor(ret, risk_free, alpha_beta);

    // 詹森指数
    jensen = cal_jensen(ret, index_ret, risk_free, alpha_beta);

    // 卡玛比率
    calmar = cal_calmar(rtn);

    // 整合后的下行标准差、欧米伽、索提诺、卡帕
    lpms = cal_omega_sortino_kappa(ret, risk_free);

    // M2
    m2 = cal_m2(ret, index_ret, risk_free);

    r = SELECT * FROM rtn a1
                 LEFT JOIN alpha_beta ON a1.entity_id = alpha_beta.entity_id
                 LEFT JOIN bmk_tracking ON a1.entity_id = bmk_tracking.entity_id
                 LEFT JOIN sharpe ON a1.entity_id = sharpe.entity_id
                 LEFT JOIN treynor ON a1.entity_id = treynor.entity_id
                 LEFT JOIN jensen ON a1.entity_id = jensen.entity_id
                 LEFT JOIN calmar ON a1.entity_id = calmar.entity_id
                 LEFT JOIN lpms ON a1.entity_id = lpms.entity_id
                 LEFT JOIN m2 ON a1.entity_id = m2.entity_id

    // 年化各数据点
    // GIPS RULE: NO annulization for data less than 1 year
    plainAnnu = get_annulization_multiple(freq);
    sqrtAnnu = sqrt(get_annulization_multiple(freq));

    r.addColumn(['std_dev_a', 'ds_dev_a', 'alpha_a', 'sharpe_a', 'sortino_a', 'jensen_a', 'track_error_a', 'info_a', 'm2_a'],
                [DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE]);

    UPDATE r 
      SET std_dev_a = std_dev * iif(price_date.month() - min_date.month() >= 11, sqrtAnnu, 1),
          ds_dev_a = ds_dev * iif(price_date.month() - min_date.month() >= 11, sqrtAnnu, 1),
          alpha_a = alpha * iif(price_date.month() - min_date.month() >= 11, plainAnnu, 1),
          sharpe_a = sharpe * iif(price_date.month() - min_date.month() >= 11, sqrtAnnu, 1),
          sortino_a = sortino * iif(price_date.month() - min_date.month() >= 11, sqrtAnnu, 1),
          jensen_a = jensen * iif(price_date.month() - min_date.month() >= 11, plainAnnu, 1),
          track_error_a = track_error * iif(price_date.month() - min_date.month() >= 11, sqrtAnnu, 1),
          info_a = info * iif(price_date.month() - min_date.month() >= 11, sqrtAnnu, 1),
          m2_a = m2 * iif(price_date.month() - min_date.month() >= 11, plainAnnu, 1);
    
    return r;
}

/*
 *   Calculate trailing 6m, ytd, 1y, 2y, 3y, 4y, 5y, 10y and since inception indicators
 *   
 *   @param: entity_info <TABLE>: basic information of entity, NEED COLUMNS entity_id, inception_date
 *   @param: ret <TABLE>: 收益表，NEED COLUMNS entity_id, price_dat, end_date, nav 
 *   @param: end_day <DATE>: 计算截止日期
 *   @param index_ret <TABLE>: historical benchmark return table, NEED COLUMNS fund_id, end_date, ret
 *   @param risk_free <TABLE>: historical risk free rate table, NEED COLUMNS fund_id, end_date, ret
 *   @param freq <CHAR>: 数据频率，d, w, m, q, s, a

 * 
 */
def cal_all_trailing_indicators(entity_info, mutable tb_ret, end_day, bmk_ret, risk_free_rate, freq) {

    r_incep = null;
    r_ytd = null;
    r_6m = null;
    r_1y = null;
    r_2y = null;
    r_3y = null;
    r_4y = null;
    r_5y = null;
    r_10y = null;
    r_ms_3y = null;
    r_ms_5y = null;
    r_ms_10y = null;

    // since inception
    if(tb_ret.size() > 0) {
        r_incep = cal_indicators(tb_ret, bmk_ret, risk_free_rate, 'm');
    }

    // ytd
    tb_ret_ytd = SELECT * FROM tb_ret WHERE end_date >= end_day.yearBegin().month();
    if(tb_ret_ytd.size() > 0) {
        r_ytd = cal_indicators(tb_ret_ytd, bmk_ret, risk_free_rate, 'm');
    }

    // trailing 6m
    tb_ret_6m = SELECT * FROM tb_ret r INNER JOIN entity_info ei ON r.entity_id = ei.entity_id
                WHERE r.end_date > end_day.month()-6 AND (end_day.month() - ei.inception_date.month()) >= 6;
    if(tb_ret_6m.size() > 0) {
        r_6m = cal_indicators(tb_ret_6m, bmk_ret, risk_free_rate, 'm');
    }
    
    // trailing 1y
    tb_ret_1y = SELECT * FROM tb_ret r INNER JOIN entity_info ei ON r.entity_id = ei.entity_id
                WHERE r.end_date > end_day.month()-12 AND (end_day.month() - ei.inception_date.month()) >= 12;
    if(tb_ret_1y.size() > 0) {
        r_1y = cal_indicators(tb_ret_1y, bmk_ret, risk_free_rate, 'm');
    }
    
    // trailing 2y
    tb_ret_2y = SELECT * FROM tb_ret r INNER JOIN entity_info ei ON r.entity_id = ei.entity_id
                WHERE r.end_date > end_day.month()-24 AND (end_day.month() - ei.inception_date.month()) >= 24;
    if(tb_ret_2y.size() > 0) {
        r_2y = cal_indicators(tb_ret_2y, bmk_ret, risk_free_rate, 'm');
    }
    
    // trailing 3y
    tb_ret_3y = SELECT * FROM tb_ret r INNER JOIN entity_info ei ON r.entity_id = ei.entity_id
                WHERE r.end_date > end_day.month()-36 AND (end_day.month() - ei.inception_date.month()) >= 36;
    if(tb_ret_3y.size() > 0) {
        r_3y = cal_indicators(tb_ret_3y, bmk_ret, risk_free_rate, 'm');
        r_ms_3y = cal_ms_return(tb_ret_3y, risk_free_rate);
    }

    // trailing 4y
    tb_ret_4y = SELECT * FROM tb_ret r INNER JOIN entity_info ei ON r.entity_id = ei.entity_id
                WHERE r.end_date > end_day.month()-48 AND (end_day.month() - ei.inception_date.month()) >= 48;
    if(tb_ret_4y.size() > 0) {
        r_4y = cal_indicators(tb_ret_4y, bmk_ret, risk_free_rate, 'm');
    }
    
    // trailing 5y
    tb_ret_5y = SELECT * FROM tb_ret r INNER JOIN entity_info ei ON r.entity_id = ei.entity_id
                WHERE r.end_date > end_day.month()-60 AND (end_day.month() - ei.inception_date.month()) >= 60;
    if(tb_ret_5y.size() > 0) {
        r_5y = cal_indicators(tb_ret_5y, bmk_ret, risk_free_rate, 'm');
        r_ms_5y = cal_ms_return(tb_ret_5y, risk_free_rate);
    }

    // trailing 10y
    tb_ret_10y = SELECT * FROM tb_ret r INNER JOIN entity_info ei ON r.entity_id = ei.entity_id
                WHERE r.end_date > end_day.month()-120 AND (end_day.month() - ei.inception_date.month()) >= 120;
    if(tb_ret_10y.size() > 0) {
        r_10y = cal_indicators(tb_ret_10y, bmk_ret, risk_free_rate, 'm');
        r_ms_10y = cal_ms_return(tb_ret_10y, risk_free_rate);
    }

    return r_incep, r_ytd, r_6m, r_1y, r_2y, r_3y, r_4y, r_5y, r_10y, r_ms_3y, r_ms_5y, r_ms_10y;
}

/*
 *   Calculate fund indicators for month-end production
 * 
 *   @param entity_type <STRING>: MF, HF
 *   @param fund_ids <STRING>: 逗号和单引号分隔的fund_id
 *   @param end_day <DATE>: 要计算的日期
 *   @param isFromNav <BOOL>: 用净值实时计算还是从表中取月收益
 *   @param isFromSQL <BOOL>: TODO: 从MySQL还是本地DolphinDB取净值/收益数据
 * 
 */
def cal_fund_indicators(entity_type, fund_ids, end_day, isFromNav) {

    very_old_date = 1990.01.01;

    fund_info = get_fund_info(fund_ids);
    fund_info.rename!('fund_id', 'entity_id');

    if(isFromNav == true) {
        // 从净值开始计算收益
        tb_ret = SELECT * FROM cal_fund_monthly_returns(entity_type, fund_ids, true) WHERE price_date <= end_day;
        tb_ret.rename!(['fund_id', 'cumulative_nav'], ['entity_id', 'nav']);
    } else {
        // 从fund_performance表里读月收益
        tb_ret = get_monthly_ret('FD', fund_ids, very_old_date, end_day, true);
        tb_ret.rename!(['fund_id'], ['entity_id']);
    }
    
    bmk_ret = SELECT fund_id, temporalParse(end_date, 'yyyy-MM') AS end_date, ret FROM get_monthly_ret('IX', "'IN00000008'", very_old_date, end_day, true); 

    risk_free_rate = SELECT fund_id, temporalParse(end_date, 'yyyy-MM') AS end_date, ret FROM get_risk_free_rate(very_old_date, end_day);

    return cal_all_trailing_indicators(fund_info, tb_ret, end_day, bmk_ret, risk_free_rate, 'm');
}

def cal_portfolio_indicators(portfolio_ids, end_day, cal_method, isFromNav) {

    very_old_date = 1990.01.01;

    portfolio_info = get_portfolio_info(portfolio_ids);
    portfolio_info.rename!('portfolio_id', 'entity_id');

    if(isFromNav == true) {
        // 从净值开始计算收益
        tb_raw_ret = SELECT * FROM cal_portfolio_return(portfolio_ids, very_old_date, cal_method) WHERE price_date <= end_day;
       
        // funky thing is you can't use "AS" for the grouping columns?
        tb_ret = SELECT portfolio_id, price_date.month(), price_date.last() AS price_date, (1+ret).prod()-1 AS ret, nav.last() AS nav
                 FROM tb_raw_ret
                 WHERE price_date <= end_day
                 GROUP BY portfolio_id, price_date.month();
        tb_ret.rename!(['portfolio_id', 'month_price_date'], ['entity_id', 'end_date']);

    } else {
        // 从pf_portfolio_performance表里读月收益
        tb_ret = get_monthly_ret('PF', portfolio_ids, very_old_date, end_day, true);
        tb_ret.rename!(['portfolio_id'], ['entity_id']);
    }

    
    bmk_ret = SELECT fund_id, temporalParse(end_date, 'yyyy-MM') AS end_date, ret FROM get_monthly_ret('IX', "'IN00000008'", very_old_date, end_day, true); 

    risk_free_rate = SELECT fund_id, temporalParse(end_date, 'yyyy-MM') AS end_date, ret FROM get_risk_free_rate(very_old_date, end_day);

    return cal_all_trailing_indicators(portfolio_info, tb_ret, end_day, bmk_ret, risk_free_rate, 'm');

    
}