first commit

codes/sample_cal_fof_holding.dos

@@ -0,0 +1,82 @@
+login(`admin, `123456)
+loadPlugin("ODBC")
+
+clearCachedModules()
+use fundit::fundCalculator
+use fundit::dataPuller
+
+end_date = 2023.07.28
+// portfolio_ids = "166002,364640,362736"
+portfolio_id = "364640"
+
+// size of rolling window
+win = 24
+// step of moving
+step = 24
+
+// get holdings
+tb_holdings = get_portfolio_holding_history(portfolio_id)
+
+// calculate current share of each holding
+tb_current_holdings = SELECT portfolio_id, end_date AS holding_date, fund_id, fund_share.sum() AS fund_share
+                      FROM tb_holdings
+                      GROUP BY portfolio_id, fund_id
+                      HAVING fund_share.sum() > 0
+
+fund_ids = tb_current_holdings.fund_id.concat("','")$STRING
+fund_ids = "'" + fund_ids + "'"
+
+tb_latest_nav = SELECT fund_id, price_date, cumulative_nav FROM get_fund_latest_nav_performance(fund_ids, true)
+
+// calculate portfolio total market value
+UPDATE tb_current_holdings a
+  SET a.market_value = round(a.fund_share * nav.cumulative_nav, 6), nav = nav.cumulative_nav
+FROM ej(tb_current_holdings a, tb_latest_nav nav, "fund_id")
+
+// calculate weighting of each holding
+tb_current_holdings = SELECT *, market_value.sum() AS total_market_value, round(market_value \ market_value.sum(), 6) AS weighting
+                      FROM tb_current_holdings
+                      CONTEXT BY portfolio_id
+
+// get weekly return of fund holdings
+//fund_ids = fund_ids + ",'MF00003PW1','MF00003PW2','IN00000008'"
+
+
+// fund_ids = fund_ids + "," + fund_pool_188.left(150*13-1)
+tb_fund_weekly_ret = SELECT ret
+                     FROM get_fund_weekly_rets(fund_ids, 1990.01.01, end_date, true).rename!(["year_week", "ret_1w"], ["date", "ret"])
+                     PIVOT BY date, fund_id
+
+// get portfolio weekly returns
+tb_portfolio_weekly_ret = get_portfolio_weekly_rets(portfolio_id, 1990.01.01, end_date, true).rename!(["year_week", "ret_1w"], ["date", "ret"])
+
+// calculate rolling RBSA weightings
+tb = cal_rolling_rbsa(tb_portfolio_weekly_ret, tb_fund_weekly_ret, false, win, step)
+
+// transform the data structure to mySQL friendly
+tb_rbsa = table(tb.size()*(tb_fund_weekly_ret.cols()-1):0, "portfolio_id" "asset_type_id" "index_id"  "effective_date" "weight", [INT, STRING, STRING, STRING, FLOAT])
+
+for( r in tb)
+{
+    if(r.status <> "solved") continue
+
+    w = r.weights.split(" ")$DOUBLE
+
+    for(i in 1..(tb_fund_weekly_ret.cols()-1))
+    {
+        tb_rbsa.tableInsert(portfolio_id$INT, "TestHolding", tb_fund_weekly_ret.colNames()[i], r.date, w[i-1])
+    }
+}
+
+SELECT * FROM tb_rbsa WHERE index_id in ('MF00003Q1A', 'MF00003T43') and effective_date = '202330' order by weight desc
+
+
+SELECT * FROM tb_current_holdings order by weighting desc
+select * from tb_portfolio_weekly_ret order by date desc
+
+t = (SELECT * FROM tb_fund_weekly_ret WHERE date > '202001').dropColumns!("date")
+m = t.matrix().corrMatrix()
+m = rename!(m, t.colNames(), t.colNames())
+m
+
+

codes/sample_cal_indicator.dos

@@ -0,0 +1,424 @@
+login(`admin, `123456)
+loadPlugin("ODBC")
+clearCachedModules()
+use fundit::fundCalculator
+use fundit::dataPuller
+use fundit::returnCalculator
+
+
+end_day = 2024.06.28
+//end_day = today()
+
+bmk_ret = get_fund_monthly_ret("'IN00000008'", 1990.01.01, end_day, true)
+risk_free_rate = get_risk_free_rate("'IN0000000M'", 1990.01.01, end_day)
+
+/* TEST CASE 1  */
+// tb_ret = get_fund_monthly_ret("'MF00003PW1','MF00003PW2'", 1990.01.01, end_day, true)
+// hedge fund test
+tb_ret = get_fund_monthly_ret("'HF000004KN','HF000103EU','HF00018WXG'", 1990.01.01, end_day, true)
+
+
+/* TEST CASE 2 
+tb_updated_funds = get_fund_list_by_nav_updatetime(2024.07.19T10:00:00)
+
+// take 1000 funds for testing
+fund_ids = "'" + tb_updated_funds.fund_id[0:1000].concat("','") + "'"
+
+tb_ret = get_fund_monthly_ret(fund_ids, 1990.01.01, end_day, false)
+*/
+
+
+/* annualized return GIPS  */
+// 0.122622 -0.163869 -0.077696
+ret_incep_a = SELECT fund_id, ret_incep_a FROM tb_ret WHERE end_date = end_day.datetimeFormat("yyyy-MM")
+ret_ytd_a = SELECT fund_id, ret_ytd_a FROM tb_ret WHERE end_date = end_day.datetimeFormat("yyyy-MM")
+
+/* annualized standard deviation  */
+// 0.210105 null null
+std_incep = SELECT ret.std() FROM tb_ret GROUP BY fund_id 
+std_incep_a = std_incep * pow(12, 0.5)
+// 0.357842 null null
+std_ytd = (SELECT ret.std() FROM tb_ret WHERE end_date >= end_day.year()$STRING + "-01" GROUP BY fund_id)
+std_ytd_a = std_ytd * pow(12, 0.5)
+// 0.357842 null null
+std_6m = (SELECT ret.std() FROM tb_ret WHERE end_date > end_day.temporalAdd(-6, "M").temporalFormat("yyyy-MM") GROUP BY fund_id) 
+std_6m_a = std_6m * pow(12, 0.5)
+// 0.270642 null null
+std_1y = (SELECT ret.std() FROM tb_ret WHERE end_date > end_day.temporalAdd(-1, "y").temporalFormat("yyyy-MM") GROUP BY fund_id) 
+std_1y_a = std_1y * pow(12, 0.5)
+// 0.249324 null null
+std_2y = (SELECT ret.std() FROM tb_ret WHERE end_date > end_day.temporalAdd(-2, "y").temporalFormat("yyyy-MM") GROUP BY fund_id)
+std_2y_a = std_2y * pow(12, 0.5)
+// 0.248138 null null
+std_3y = (SELECT ret.std() FROM tb_ret WHERE end_date > end_day.temporalAdd(-3, "y").temporalFormat("yyyy-MM") GROUP BY fund_id)
+std_3y_a = std_3y * pow(12, 0.5)
+// 0.242779 null null
+std_4y = (SELECT ret.std() FROM tb_ret WHERE end_date > end_day.temporalAdd(-4, "y").temporalFormat("yyyy-MM") GROUP BY fund_id)
+std_4y_a = std_4y * pow(12, 0.5)
+// 0.224387 null null
+std_5y = (SELECT ret.std() FROM tb_ret WHERE end_date > end_day.temporalAdd(-5, "y").temporalFormat("yyyy-MM") GROUP BY fund_id)
+std_5y_a = std_5y * pow(12, 0.5)
+// 0.210569 null null
+std_10y = (SELECT ret.std() FROM tb_ret WHERE end_date > end_day.temporalAdd(-10, "y").temporalFormat("yyyy-MM") GROUP BY fund_id)
+std_10y_a = std_10y * pow(12, 0.5)
+
+/* annualized downside stdev  numbers are slightly off !
+   NOTE: not sure why cnt[0] need to minus 1 copied from Java implementation*/
+// risk_free_rate = 0.001208
+// 0.130429 null null
+t = SELECT *, count(fund_id) AS cnt FROM tb_ret CONTEXT BY fund_id
+ds_std_dev_incep = SELECT t.fund_id, (sum2(t.ret - rfr.ret) / (t.cnt[0]-1)).pow(0.5) AS ds_std_dev 
+                   FROM t 
+                   INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+                   WHERE t.ret < rfr.ret
+                   GROUP BY t.fund_id
+ds_std_dev_incep_a = ds_std_dev_incep * pow(12, 0.5)
+
+// 0.277208 null null
+t = SELECT *, count(fund_id) AS cnt FROM tb_ret WHERE end_date >= end_day.year()$STRING + "-01"  CONTEXT BY fund_id
+ds_std_dev_ytd = SELECT fund_id, (sum2(ret - rfr.ret) / (t.cnt[0]-1)).pow(0.5) AS ds_std_dev
+                 FROM t
+                 INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+                 WHERE t.ret < rfr.ret
+                 GROUP BY fund_id 
+ds_std_dev_ytd_a = ds_std_dev_ytd * pow(12, 0.5)
+
+
+/* beta  */
+//  0.530483 null null
+t = SELECT t.fund_id, t.end_date, t.ret, bmk.ret AS ret_bmk
+     FROM tb_ret t
+     INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
+
+beta_incep = SELECT ret.beta(ret_bmk) AS beta FROM t GROUP BY fund_id
+
+// 1.819822 null null
+t = SELECT t.fund_id, t.end_date, t.ret, bmk.ret AS ret_bmk
+     FROM tb_ret t
+     INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
+     WHERE t.end_date >= end_day.datetimeFormat("yyyy-01")
+
+beta_ytd =SELECT ret.beta(ret_bmk) AS beta FROM t GROUP BY fund_id
+
+/* annualized alpha -- numbers are off ！ because Java doesn't substract risk free rate */
+
+ //  0.110500 null null
+t = SELECT t.fund_id, t.end_date, t.ret, bmk.ret AS ret_bmk
+      FROM tb_ret t
+      INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
+
+alpha_incep = SELECT t.fund_id, (t.ret - rfr.ret).mean() - beta.beta[0] * (t.ret_bmk - rfr.ret).mean() AS alpha
+              FROM t 
+              INNER JOIN beta_incep beta ON t.fund_id = beta.fund_id
+              INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+              GROUP BY t.fund_id
+alpha_incep_a = alpha_incep * 12
+
+ // -0.11019 null null
+t = SELECT t.fund_id, t.end_date, t.ret, bmk.ret AS ret_bmk
+    FROM tb_ret t
+    INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
+    WHERE t.end_date >= end_day.datetimeFormat("yyyy-01")
+
+ alpha_ytd  = SELECT t.fund_id, (t.ret - rfr.ret).mean() - beta.beta[0] * (t.ret_bmk - rfr.ret).mean() AS alpha
+              FROM t 
+              INNER JOIN beta_ytd beta ON t.fund_id = beta.fund_id
+              INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+              GROUP BY t.fund_id
+alpha_ytd_a = alpha_ytd * 12
+
+/* win rate -- numbers are slightly way off ！effective data count issue? */
+
+ // 0.585185 null null
+ t = SELECT t.fund_id, t.end_date, t.ret, bmk.ret AS ret_bmk, count(fund_id) AS cnt
+     FROM tb_ret t
+     INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
+     CONTEXT BY t.fund_id
+
+winrate_incep = SELECT fund_id, count(fund_id) \ cnt[0] AS winrate
+                FROM t
+                WHERE ret > ret_bmk
+                GROUP BY fund_id
+
+// 0.666667 null null
+ t = SELECT t.fund_id, t.end_date, t.ret, bmk.ret AS ret_bmk, count(fund_id) AS cnt
+     FROM tb_ret t
+     INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
+     WHERE t.end_date >= end_day.datetimeFormat("yyyy-01")
+     CONTEXT BY t.fund_id
+
+winrate_ytd = SELECT fund_id, count(fund_id) \ cnt[0] AS winrate
+              FROM t
+              WHERE ret > ret_bmk
+              GROUP BY fund_id
+
+/* skewness */
+// -0.090416
+skewness_incep = SELECT fund_id, ret.skew(false) AS skewness FROM tb_ret GROUP BY fund_id
+// -0.513185
+skewness_ytd = SELECT fund_id, ret.skew(false) AS skewness FROM tb_ret WHERE end_date >= end_day.datetimeFormat("yyyy-01") GROUP BY fund_id
+
+/* kurtosis */
+// 0.527901
+kurtosis_incep = SELECT fund_id, ret.kurtosis(false) - 3 AS kurtosis FROM tb_ret GROUP BY fund_id
+// 0.961329
+kurtosis_ytd = SELECT fund_id, ret.kurtosis(false) - 3 AS kurtosis FROM tb_ret WHERE end_date >= end_day.datetimeFormat("yyyy-01") GROUP BY fund_id
+
+/* worst month */
+// --0.172580
+wrst_month_incep = SELECT ret.min() AS ret FROM tb_ret GROUP BY fund_id
+// -0.172580
+wrst_month_ytd = SELECT ret.min() AS ret FROM tb_ret WHERE end_date >= end_day.datetimeFormat("yyyy-01") GROUP BY fund_id
+
+/* longest down month*/
+
+/* max drawdown */
+// 0.305496 0.5621 0.120200
+drawdown_incep = SELECT fund_id, max( 1 - nav \ nav.cummax() ) AS drawdown FROM tb_ret GROUP BY fund_id
+
+// 0.05645 0 0.06553
+drawdown_ytd = SELECT fund_id, max( 1 - nav \ nav.cummax() ) AS drawdown FROM tb_ret WHERE end_date >= end_day.datetimeFormat("yyyy-01") GROUP BY fund_id
+
+/* sharpe ratio */
+// 0.568739
+sharpe_incep = SELECT t.fund_id, (t.ret - rfr.ret).mean() / std.std_ret[0] AS sharpe
+               FROM tb_ret t
+               INNER JOIN std_incep std ON t.fund_id = std.fund_id
+               INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+               GROUP BY t.fund_id
+sharpe_incep_a = sharpe_incep * pow(12, 0.5)
+
+// -0.186892
+sharpe_ytd   = SELECT t.fund_id, (t.ret - rfr.ret).mean() / std.std_ret[0] AS sharpe
+               FROM tb_ret t
+               INNER JOIN std_ytd std ON t.fund_id = std.fund_id
+               INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+               WHERE t.end_date >= end_day.datetimeFormat("yyyy-01")
+               GROUP BY t.fund_id
+sharpe_ytd_a = sharpe_ytd * pow(12, 0.5)
+
+// -0.568024
+sharpe_1y =     SELECT t.fund_id, (t.ret - rfr.ret).mean() / std.std_ret[0] AS sharpe
+                FROM tb_ret t
+                INNER JOIN std_1y std ON t.fund_id = std.fund_id
+                INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+                WHERE t.end_date > end_day.temporalAdd(-1, "y").temporalFormat("yyyy-MM")
+                GROUP BY t.fund_id
+sharpe_1y_a = sharpe_1y * pow(12, 0.5)
+
+// -0.131570
+sharpe_2y =     SELECT t.fund_id, (t.ret - rfr.ret).mean() / std.std_ret[0] AS sharpe
+                FROM tb_ret t
+                INNER JOIN std_2y std ON t.fund_id = std.fund_id
+                INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+                WHERE t.end_date > end_day.temporalAdd(-2, "y").temporalFormat("yyyy-MM")
+                GROUP BY t.fund_id
+sharpe_2y_a = sharpe_2y * pow(12, 0.5)
+
+
+/* sortino ratio  NOTE: sortino is LPM2*/
+
+// 0.916167
+sortino_incep = SELECT t.fund_id, (t.ret - rfr.ret ).mean() / std.ds_std_dev[0] AS sortino
+                FROM tb_ret t
+                INNER JOIN ds_std_dev_incep std ON t.fund_id = std.fund_id
+                INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+                GROUP BY t.fund_id
+sortino_incep_a = sortino_incep * pow(12, 0.5)
+
+// -0.241256
+sortino_ytd   = SELECT t.fund_id, (t.ret - rfr.ret ).mean() / std.ds_std_dev[0] AS sortino
+                FROM tb_ret t
+                INNER JOIN ds_std_dev_ytd std ON t.fund_id = std.fund_id
+                INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+                WHERE t.end_date >= end_day.datetimeFormat("yyyy-01")
+                GROUP BY t.fund_id
+sortino_ytd_a = sortino_ytd * pow(12, 0.5)
+
+/* sortino MAR  -- what's the MAR number? */
+
+
+
+/* treynor ratio  ytd numbers are way off ! because current java calcuation is not follow GIPS annualization rule*/
+// 0.195812
+t= SELECT *, count(fund_id) AS cnt FROM tb_ret t INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date CONTEXT BY t.fund_id
+treynor_incep = SELECT t.fund_id, ((1 + t.ret).prod().pow(12\iif(t.cnt[0]<12, 12, t.cnt[0])) - (1 + t.rfr_ret).prod().pow(12\iif(t.cnt[0]<12, 12, t.cnt[0]))) / beta.beta[0] AS treynor
+                FROM t
+                INNER JOIN beta_incep beta ON t.fund_id = beta.fund_id
+                GROUP BY t.fund_id
+
+// -0.064390
+t= SELECT *, count(fund_id) AS cnt FROM tb_ret t INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date WHERE t.end_date >= end_day.datetimeFormat("yyyy-01") CONTEXT BY t.fund_id
+treynor_ytd =   SELECT t.fund_id, ((1 + t.ret).prod().pow(12\iif(t.cnt[0]<12, 12, t.cnt[0])) - (1 + t.rfr_ret).prod().pow(12\iif(t.cnt[0]<12, 12, t.cnt[0]))) / beta.beta[0] AS treynor
+                FROM t
+                INNER JOIN beta_ytd beta ON t.fund_id = beta.fund_id
+                GROUP BY t.fund_id
+
+
+/* jensen's alpha  numbers are slightly off ! */
+// 0.101781
+jensen_incep = SELECT t.fund_id, t.ret.mean() - rfr.ret.mean() - beta.beta[0] * (bmk.ret.mean() - rfr.ret.mean()) AS jensen
+               FROM tb_ret t
+               INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
+               INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+               INNER JOIN beta_incep beta ON t.fund_id = beta.fund_id
+               GROUP BY t.fund_id
+jensen_incep_a = jensen_incep * 12
+
+// -0.098310
+jensen_ytd   = SELECT t.fund_id, t.ret.mean() - rfr.ret.mean() - beta.beta[0] * (bmk.ret.mean() - rfr.ret.mean()) AS jensen
+               FROM tb_ret t
+               INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
+               INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+               INNER JOIN beta_ytd beta ON t.fund_id = beta.fund_id
+               WHERE t.end_date >= end_day.datetimeFormat("yyyy-01")
+               GROUP BY t.fund_id
+jensen_ytd_a = jensen_ytd * 12
+
+/* calmar ratio   numbers are way off !  */
+// 0.376378 -0.279307 -0.81280
+calmar_incep = SELECT fund_id, ar.ret_incep_a \ dd.drawdown AS calmar
+               FROM ret_incep_a ar
+               INNER JOIN drawdown_incep dd ON ar.fund_id = dd.fund_id
+// 2.567034 999999. -0.81280
+calmar_ytd =   SELECT fund_id, ar.ret_ytd_a \ dd.drawdown  AS calmar
+               FROM ret_ytd_a ar
+               INNER JOIN drawdown_ytd dd ON ar.fund_id = dd.fund_id
+
+/* omega ratio   numbers are off ! could because Java uses annualized returns and cnt-1  NOTE: omega is LPM1 */
+// 1.471981
+t0 = SELECT *, fund_id.count() AS cnt FROM tb_ret t INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date CONTEXT BY t.fund_id
+       
+t1 = SELECT fund_id, (ret - rfr_ret).sum() \ cnt[0] AS ex_ret
+     FROM t0
+     WHERE ret > rfr_ret
+     GROUP BY fund_id
+t2 = SELECT fund_id, (rfr_ret - ret).sum() \ cnt[0] AS ds_ret
+     FROM t0
+     WHERE ret < rfr_ret
+     GROUP BY fund_id
+
+omega_incep = SELECT t1.fund_id,  t1.ex_ret \ t2.ds_ret AS omega
+              FROM t1
+              INNER JOIN t2 ON t1.fund_id = t2.fund_id
+
+// 0.790864
+t0 = SELECT *, fund_id.count() AS cnt FROM tb_ret t INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date WHERE t.end_date >= end_day.datetimeFormat("yyyy-01") CONTEXT BY t.fund_id
+       
+t1 = SELECT fund_id, (ret - rfr_ret).mean() AS ex_ret
+     FROM t0
+     GROUP BY fund_id
+t2 = SELECT fund_id, (rfr_ret - ret).sum() \ cnt[0] AS ds_ret
+     FROM t0
+     WHERE ret < rfr_ret
+     GROUP BY fund_id
+
+omega_ytd =   SELECT t1.fund_id,  1 + t1.ex_ret \ t2.ds_ret AS omega
+              FROM t1
+              INNER JOIN t2 ON t1.fund_id = t2.fund_id
+
+/* kappa ratio NOTE: kappa is LMP3  numbers are off ! Java's implementation could be very wrong */ 
+
+// 0.848648
+t0 = SELECT *, fund_id.count() AS cnt FROM tb_ret t INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date CONTEXT BY t.fund_id
+t1 = SELECT fund_id, (ret - rfr_ret).mean() AS ex_ret
+     FROM t0
+     GROUP BY fund_id
+t2 = SELECT fund_id, (rfr_ret - ret).sum3() \ cnt[0] AS ds_ret
+     FROM t0
+     WHERE ret < rfr_ret
+     GROUP BY fund_id
+
+kappa_incep = SELECT t1.fund_id,  1 + t1.ex_ret \ t2.ds_ret.pow(1\3) AS kappa
+              FROM t1
+              INNER JOIN t2 ON t1.fund_id = t2.fund_id
+// -0.501813
+t0 = SELECT *, fund_id.count() AS cnt FROM tb_ret t INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date WHERE t.end_date >= end_day.datetimeFormat("yyyy-01") CONTEXT BY t.fund_id
+t1 = SELECT fund_id, (ret - rfr_ret).mean() AS ex_ret
+     FROM t0
+     GROUP BY fund_id
+t2 = SELECT fund_id, (rfr_ret - ret).sum3() \ cnt[0] AS ds_ret
+     FROM t0
+     WHERE ret < rfr_ret
+     GROUP BY fund_id
+
+kappa_ytd =   SELECT t1.fund_id,  1+ t1.ex_ret \ t2.ds_ret.pow(1\3) AS kappa
+              FROM t1
+              INNER JOIN t2 ON t1.fund_id = t2.fund_id
+
+/* tracking error */
+// 0.203268
+track_error_incep = SELECT fund_id,  (t.ret - bmk.ret).std() AS track_error
+                    FROM tb_ret t
+                    INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
+                    GROUP BY t.fund_id
+track_error_incep_a = track_error_incep * pow(12, 0.5)
+// 0.193291
+track_error_ytd =   SELECT fund_id,  (t.ret - bmk.ret).std() AS track_error
+                    FROM tb_ret t
+                    INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
+                    WHERE t.end_date >= end_day.datetimeFormat("yyyy-01")
+                    GROUP BY t.fund_id
+track_error_ytd_a = track_error_ytd * pow(12, 0.5)
+
+/* information ratio  numbers are way off! */
+// 5.472180
+info_incep = SELECT fund_id, (t.ret - bmk.ret).mean() / (t.ret - bmk.ret).std() AS info
+             FROM tb_ret t
+             INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
+             GROUP BY t.fund_id
+info_incep_a = info_incep * pow(12, 0.5)
+
+
+/* modigliani  = sharpe * std(benchmark) + risk_free_rate )*/ 
+// 0.141025
+m2_incep = SELECT t.fund_id, (t.ret - rfr.ret).mean() / t.ret.std() * bmk.ret.std() + rfr.ret.mean()
+           FROM tb_ret t
+           INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
+           INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+           GROUP BY t.fund_id
+m2_incep_a = m2_incep * 12
+// -0.020145
+m2_ytd   = SELECT t.fund_id, (t.ret - rfr.ret).mean() / t.ret.std() * bmk.ret.std() + rfr.ret.mean()
+           FROM tb_ret t
+           INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
+           INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+           WHERE t.end_date >= end_day.datetimeFormat("yyyy-01")
+           GROUP BY t.fund_id
+m2_ytd_a = m2_ytd * 12
+
+/* historical var  numbers are off ! due to different implementations */
+CONFIDENCE_LEVAL = 95
+// 0.093206
+var_incep = SELECT fund_id, - ret.percentile(100-CONFIDENCE_LEVAL) AS var
+            FROM tb_ret
+            GROUP BY fund_id
+// 0.162951
+var_2y    = SELECT fund_id, - ret.percentile(100-CONFIDENCE_LEVAL) AS var
+            FROM tb_ret
+            WHERE end_date > end_day.temporalAdd(-2, "y").datetimeFormat("yyyy-MM")
+            GROUP BY fund_id
+
+
+
+/* historical cvar  numbers are slightly off ! due to different implementations of var */
+// 0.129327
+cvar_incep = SELECT t.fund_id, - t.ret.mean() AS cvar
+             FROM tb_ret t
+             INNER JOIN var_incep var ON t.fund_id = var.fund_id 
+             WHERE t.ret < - var.var
+             GROUP BY t.fund_id
+
+// 0.17258
+cvar_2y    = SELECT t.fund_id, - t.ret.mean() AS cvar
+             FROM tb_ret t
+             INNER JOIN var_2y var ON t.fund_id = var.fund_id 
+             WHERE end_date > end_day.temporalAdd(-2, "y").datetimeFormat("yyyy-MM")
+                AND t.ret < - var.var
+             GROUP BY t.fund_id
+
+/* SMDD var */
+
+
+/* stutzer index, used by Tian Xiang rating */ 
+// stutzer = (tb_ret - risk_free_rate) \ cvar

codes/sample_cal_return.dos

@@ -0,0 +1,31 @@
+login(`admin, `123456)
+loadPlugin("ODBC")
+clearCachedModules()
+use fundit::fundCalculator
+use fundit::dataPuller
+use fundit::returnCalculator
+
+// TEST CASE 1
+fund_ids = "'HF000004KN','HF00018WXG','HF000103EU'"
+
+// TEST CASE 2 取有净值更新的所有私募基金
+tb_fund_list = get_fund_list_by_nav_updatetime(2024.07.19T10:00:00)
+fund_ids = tb_fund_list.fund_id.concat("','")$STRING
+fund_ids = "'" + fund_ids$STRING + "'"
+
+// 取基金净值
+tb_nav = get_hedge_fund_nav_by_price_date(fund_ids, 1990.01.01, true)
+save_hedge_fund_nav_to_local(tb_nav)
+
+// 计算基金月收益
+tb_fund_performance = cal_hedge_fund_returns(fund_ids, false)
+save_table(tb_fund_performance, "mfdb.fund_performance", false)
+
+// 计算基金周收益
+tb_fund_performance_weekly = cal_hedge_fund_weekly_returns(fund_ids, false)
+save_table(tb_fund_performance_weekly, "mfdb.fund_performance_weekly", false)
+
+// 计算基金最新收益
+tb_fund_latest_nav_performance = cal_hedge_fund_latest_returns(fund_ids, false)
+save_table(tb_fund_latest_nav_performance, "mfdb.fund_latest_nav_performance", false)
+

codes/sample_rbsa.dos

@@ -0,0 +1,202 @@
+login(`admin, `123456)
+loadPlugin("ODBC")
+go
+
+clearCachedModules()
+use fundit::fundCalculator
+use fundit::dataPuller
+
+
+/* Sample 1: 单基金或组合RBSA  ------------------------------ */
+
+IN00000008 = (0.007075 -0.007000 -0.002121 0.008571 -0.025775 0.018936 0.012031 0.005613 0.017171 0.003189 -0.020797 -0.005986);
+IN00000077 = (-0.002598 0.001488 0.001404 0.001031 0.002389 0.002999 -0.000246 -0.000533 -0.000656 0.001313 0.001352 0.001514);
+IN0000007G =(0.000373 0.000367 0.000379 0.000272 0.000461 0.000355 0.000349 0.000195 0.000478 0.000331 0.000330 0.000324);
+IN0000009M = (0.010477 0.010086 0.019914 0.027885 0.061534 0.011593 -0.032239 -0.003192 0.011932 0.024807 0.006475 0.014500);
+mt0 = table(IN00000008, IN00000077, IN0000007G, IN0000009M);
+MF00003PW1 = (0.017450 0.002639 -0.026316 -0.005405 -0.013587 -0.001377 0.024828 0.014805 0.011936 -0.003932 -0.021053 0.006721);
+
+res = cal_rbsa(MF00003PW1, mt0, false)
+
+// lb = 0 时仍有微小负数，估计可以换成0
+// 不清除后面的12维数字是干啥的
+beta = res[1][0:4]
+print(beta)
+
+/* Sample 1 end  --------------------------------------------- */
+
+
+/* Sample 2 start ------------------------------ */
+
+// pull all historical weekly returns from mysql, then save it to local
+// tb_all = get_fund_weekly_rets(null, 2021.07.01, null, true)
+// save_table(tb_all, "mfdb.fund_performance_weekly", false)
+
+// pull historical weekly returns from pre-saved local table
+// NOTE: if the table doesn't exist, run the commented code above
+tb_all_weekly_ret = load_table_from_local("fundit", "mfdb.fund_performance_weekly")
+
+// get index weekly return based on RBSA asset_type
+def get_standard_rbsa_index_return(asset_type_id, start_date, end_date) {
+
+    index_ids = ""
+
+    // asset allocation: stock-bond-cash-gold
+	if(asset_type_id == "AS0000005Q")
+	    index_ids = "'IN00000008','IN00000077','IN0000007G','IN0000009M'"
+    // asset allocation: stock-bond-cash-gold
+	else if(asset_type_id == "Large4Assets" )
+        index_ids = "'IN00000008','IN00000077','IN0000007G','IN0000009M'"
+    // Fundit style: large-small-cash
+	else if(asset_type_id == "Cap3Style" )
+        index_ids = "'FA00000WKG','FA00000WKH','IN0000007G'"
+    // Fundit sector: 防守-周期-敏感-科技-cash
+	else if(asset_type_id == "CSI5" )
+        index_ids = "'FA00000VML','FA00000VMM','FA00000VMN','FA00000VMO','IN0000007G'"
+    // CNI style: LG-LV-SG-SV-cash
+	else if(asset_type_id == "CNI5Style" )
+        index_ids = "'IN0000000S','IN0000000T','IN0000000U','IN0000000V','IN0000007G'"
+    // CNI style: LG-LV-SG-SV-MG-MV-cash
+	else if(asset_type_id == "CNI7Style" )
+        index_ids = "'IN0000000S','IN0000000T','IN0000000U','IN0000000V','IN0000000W', 'IN0000000X','IN0000007G'"
+    // CSI old sector: 材料-电信-工业-公用-金融-可选-能源-消费-信息-医药-cash
+	else if(asset_type_id == "CSI11" )
+        index_ids = "'IN0000000Y','IN0000000Z','IN00000010','IN00000011','IN00000012','IN00000013','IN00000014','IN00000015','IN00000016','IN00000017','IN0000007G'"
+    // bond type: gov-cash-cnvt-corp
+	else if(asset_type_id == "BondType" )
+        index_ids = "'IN0000007A','IN0000007G','IN0000008J','IN000002CM'"
+
+    ret = get_index_weekly_rets(index_ids, start_date, end_date)
+
+    return ret
+
+}
+
+
+win = 24
+step = 24
+
+// pick the fund to be compared with
+the_fund = 'MF00003PW1'
+the_fund_ret = (SELECT * FROM tb_all_weekly_ret WHERE year_week >= '202211' AND fund_id = the_fund).ret_1w
+
+// get all mutual fund ids for testing
+fund_ids = (SELECT DISTINCT fund_id FROM tb_all_weekly_ret).fund_id
+
+// all rbsa results will be saved here
+tb_all_rbsa = table(fund_ids.size()*(the_fund_ret.size()):0, "fund_id" "date" "asset_type_id" "weights", [STRING, STRING, STRING, STRING])
+
+// all the distances will be saved here
+tb_dis = table(fund_ids.size():0, "fund_id" "date" "asset_type_id" "dist", [STRING, STRING, STRING, DOUBLE])
+
+// asset_type_id = "Cap3Style"
+asset_type = "CSI5"
+tb_index_raw_ret = get_standard_rbsa_index_return(asset_type, 2021.07.01, 2024.07.26)
+
+// transform the data structure for rbsa calculation
+tb_index_weekly_ret = SELECT ret_1w
+                      FROM tb_index_raw_ret
+                      PIVOT BY year_week, index_id
+tb_index_weekly_ret.rename!("year_week", "date")
+
+
+// loop thru the fund list and calculate historical rbsa
+// NOTE: it takes about 6 minutes for 20,000 mutual funds with 3 year history
+for(f_id in fund_ids) {
+
+    ret = SELECT year_week AS date, ret_1w AS ret
+          FROM tb_all_weekly_ret
+          WHERE fund_id = f_id
+          ORDER BY year_week
+
+	// calculate historical rbsa
+	tb = cal_rolling_rbsa(ret, tb_index_weekly_ret, false, win, step)
+
+	INSERT INTO tb_all_rbsa 
+      SELECT f_id, date, asset_type, weights
+      FROM tb
+      WHERE status = "solved"
+}
+
+SELECT * FROM tb_all_rbsa WHERE fund_id = the_fund
+
+// tb_all_weekly_ret = null
+
+// save to local db, just for the sake of saving time to run rbsa again
+save_table(tb_all_rbsa, "mfdb.fund_rbsa_weekly", false)
+tb_all_rbsa = null
+
+asset_type = "CSI5"
+the_dates = (SELECT DISTINCT date FROM tb_all_rbsa WHERE fund_id = the_fund AND asset_type_id = asset_type).date
+
+
+// loop thru all the dates
+for(d in the_dates) {
+
+    // the rbsa weights of target fund of one date
+    the_weights = (SELECT * FROM tb_all_rbsa WHERE fund_id = the_fund AND date = d AND asset_type_id = asset_type).weights[0]
+
+    // all rbsa results with the same date
+    t = SELECT * FROM tb_all_rbsa WHERE date = d AND asset_type_id = asset_type
+
+    // calculate the distance between the target fund and any fund
+    for(r in t) {
+        // euclidean distance
+        tb_dis.tableInsert(r.fund_id, d, asset_type, ((the_weights.split(" ")$DOUBLE).euclidean(r.weights.split(" ")$DOUBLE)).round(4))
+        // tanimoto distance
+        // tb_dis.tableInsert(r.fund_id, d, asset_type, ((the_weights.split(" ")$DOUBLE).tanimoto(r.weights.split(" ")$DOUBLE)).round(4))
+    }
+}
+
+select * from tb_dis where fund_id = the_fund
+
+n = the_dates.size()
+
+select fund_id, avg
+from (
+  select fund_id, count(dist) as cnt, avg(dist) as avg
+  from tb_dis
+  where fund_id <> the_fund
+    and asset_type_id = "Large4Assets"
+  group by fund_id )
+order by avg
+limit 500
+
+select fund_Id, avg from (
+  select fund_id, sum(dist) as total_dist, count(dist) as cnt, avg(dist) as avg
+  from tb_dis 
+  where fund_id <> the_fund and asset_type_id <> "Cap3Style"
+  group by fund_id )
+where cnt = n*2
+order by avg
+limit 500
+
+
+select * from tb_all_rbsa where fund_id IN ('MF00003PW1', 'MF00006EQ6', 'MF00003QZR','MF000074ZM','MF00006FQ5')
+t = select ret_1w from tb_all_weekly_ret where fund_id IN ('MF00003PW1', 'MF00006EQ6', 'MF00003QZR','MF000074ZM','MF00006FQ5') and year_week >= '202211' pivot by year_week, fund_id
+plot([t.MF00003PW1, t.MF00006EQ6], t.year_week, , LINE)
+plot([t.MF00003PW1, t.MF00003QZR], t.year_week, , LINE)
+plot([t.MF00003PW1, t.MF000074ZM], t.year_week, , LINE)
+plot([t.MF00003PW1, t.MF00006FQ5], t.year_week, , LINE)
+
+
+// calculate correlation
+t = (SELECT ret_1w FROM tb_all_weekly_ret WHERE year_week >= '202211' PIVOT BY year_week, fund_id ).dropColumns!("year_week")
+v_cols = t.colNames()
+m = matrix(t)
+tb_corr = table(fund_ids.size():0, "fund_id" "corr", [STRING, DOUBLE])
+i = 0
+for(c in m) {
+
+    if(c.dropna().size() == the_fund_ret.rows()) {
+        tb_corr.tableInsert(v_cols[i], the_fund_ret.corr(c).round(6))
+    }
+
+    i += 1
+}
+
+select * from tb_corr order by corr desc limit 500
+
+
+
+/* Sample 2 end  --------------------------------------------- */