上传文件至 'modules'

modules/dataPuller.dos

@@ -189,11 +189,11 @@ def get_fund_monthly_ret(fund_ids, start_date, end_date, isFromMySQL) {
 
         ret_table_name = "mfdb.fund_performance"
         
-        s_query = "SELECT fund_id, end_date, ret_1m AS ret, cumulative_nav AS nav, ret_ytd_a, ret_incep_a
+        s_query = "SELECT fund_id, end_date, price_date, ret_1m AS ret, cumulative_nav AS nav, ret_ytd_a, ret_incep_a
                    FROM " + ret_table_name + "
                    WHERE fund_id IN (" + fund_ids + ")
                       AND isvalid = 1
-                      AND ret_1m IS NOT NULL
+                      AND cumulative_nav > 0
                       AND end_date BETWEEN '" + yyyymm_start + "' AND '" + yyyymm_end + "'
                    ORDER BY fund_id, end_date"
      
@@ -268,18 +268,34 @@ def get_fund_latest_nav_performance(fund_ids, isFromMySQL) {
 
 /*
  * 取私募基金净值
+ * 
+ * 
+ * 
+ * Create: 202408                                                    Joey
+ *                 TODO: add isvalid and nav > 0 for local version
+ * 
  *
- * get_hedge_fund_nav_by_price_date("'HF000004KN','HF00018WXG'", 2024.05.01, true)
+ * Example: get_hedge_fund_nav_by_price_date("'HF000004KN','HF00018WXG'", 2024.05.01, true)
  */
 def get_hedge_fund_nav_by_price_date(fund_ids, price_date, isFromMySQL) {
 
+    s_fund_ids = '';
+    
+    // 判断输入的 fund_ids 是字符串标量还是向量
+    if ( fund_ids.form() == 0 ) {
+    	s_fund_ids = fund_ids;
+    } else {
+    	s_fund_ids = "'" + fund_ids.concat("','") + "'";
+    }
+
+
     if(isFromMySQL == true) {
 
         nav_table_name = "mfdb.nav"
     
         s_query = "SELECT fund_id, price_date, cumulative_nav
                    FROM " + nav_table_name + "
-                   WHERE fund_id IN (" + fund_ids + ")
+                   WHERE fund_id IN (" + s_fund_ids + ")
                      AND isvalid = 1
                      AND cumulative_nav > 0
                      AND price_date >= '" + price_date$STRING + "'
@@ -375,15 +391,27 @@ def get_fund_info(fund_ids) {
 /*
  * 取私募基金净值更新信息, 返回基金及其净值更新的最早净值日期
  *
- * get_fund_list_by_nav_updatetime(2024.07.19T10:00:00)
+ * @param fund_ids: fund_id STRING VECTOR
+ * @param update_time: all updates after this time
+ *
+ * Example: get_fund_list_by_nav_updatetime(null, 2024.07.19T10:00:00)
+ * 
  */
-def get_fund_list_by_nav_updatetime(updatetime) {
+def get_fund_list_by_nav_updatetime(fund_ids, updatetime) {
 
-    s_query = "SELECT fi.fund_id, MIN(nav.price_date) AS price_date
-               -- fi.inception_date, fi.primary_benchmark_id AS benchmark_id, IFNULL(fi.initial_unit_value, 1) AS ini_value, 
+    s_fund_sql = '';
+    // 这里要用 isVoid， 因为 isNull对向量返回的是布尔向量
+    if (! isVoid(fund_ids)){
+    	s_fund_ids = fund_ids.concat("','");
+    	s_fund_sql = " AND fi.fund_id IN ('" + s_fund_ids + "')";
+    }
+    
+    s_query = "SELECT fi.fund_id, MIN(nav.price_date) AS price_date,
+                      fi.inception_date, fi.primary_benchmark_id AS benchmark_id, IFNULL(fi.initial_unit_value, 1) AS ini_value
                FROM mfdb.fund_information fi
                INNER JOIN mfdb.nav ON fi.fund_id = nav.fund_id
-               WHERE fi.isvalid = 1
+               WHERE fi.isvalid = 1" +
+                 s_fund_sql + "
                  AND nav.cumulative_nav > 0
                  AND nav.updatetime >= '" + updatetime + "'
                GROUP BY fi.fund_id
@@ -433,4 +461,27 @@ def get_portfolio_holding_history(portfolio_ids) {
 
     return t
 
-}
+}
+
+/*
+ * 取私募基金用于月末 fund_performance 表更新的净值
+ * 
+ * @param fund_ids: 逗号分隔的ID字符串, 每个ID都有''
+ * @param month_end: 月末日期字符串  YYYY-MM
+ * 
+ * 
+ */
+def get_nav_for_hedge_fund_performance(fund_ids, month_end) {
+   
+    s_query = "CALL pfdb.sp_get_nav_for_fund_performance(" + fund_ids + ", '" + month_end + "', 1);"
+
+    conn = connect_mysql()
+
+    t = odbc::query(conn, s_query)
+
+    conn.close()
+
+    return t
+
+
+}

modules/indicatorCalculator.dos

@@ -0,0 +1,316 @@
+module fundit::indicatorCalculator
+
+/*
+ *  Annulized multiple
+ */
+def get_annulization_multiple(freq) {
+
+  ret = 1;
+  
+  if (freq == 'd') {
+  	ret = 252; // We have differences here between Java and DolphinDB, Java uses 365.25 days
+  } else if (freq == 'w') {
+  	ret = 52;
+  } else if (freq == 'm') {
+  	ret = 12;
+  } else if (freq == 'q') {
+  	ret = 4;
+  } else if (freq == 's') {
+  	ret = 2;
+  } else if (freq == 'a') {
+  	ret = 1;
+  }
+  
+  return ret;
+}
+
+
+/*
+ *     Trailing Return, Standard Deviation, Skewness, Kurtosis, Max Drawdown, VaR, CVaR
+ *     @param ret: 收益表，需要有 entity_id, price_dat, end_date, nav 
+ *     @param freq: 数据频率，d, w, m, q, s, a
+ *     
+ *     Create:  20240904                                                                     Joey
+ *                       TODO: var and cvar are silightly off compared with Java version
+ *     
+ */
+def cal_basic_performance(ret) {
+
+    t =	SELECT entity_id, max(end_date) AS end_date, max(price_date) AS price_date, min(price_date) AS min_date,
+	           (nav.last() \ nav.first() - 1).round(6) AS trailing_ret,
+	           iif(price_date.max().month()-price_date.min().month()>12,
+	               (nav.last() \ nav.first()).pow(365 \(max(price_date) - min(price_date)))-1, 
+	               (nav.last() \ nav.first() - 1)).round(6) AS trailing_ret_a,
+               ret.std() AS std_dev,
+               ret.skew(false) AS skewness,
+               ret.kurtosis(false) - 3 AS kurtosis,
+               ret.min() AS wrst_month,
+               max( 1 - nav \ nav.cummax() ) AS drawdown
+        FROM ret
+	    GROUP BY entity_id;
+
+    // var & cvar require return NOT NULL
+    // NOTE: DolphinDB supports 4 different ways: normal, logNormal, historical, monteCarlo. we use historical
+    t1 = SELECT entity_id, max(end_date) AS end_date, max(price_date) AS price_date,
+                ret.VaR('historical', 0.95) AS var,
+                ret.CVaR('historical', 0.95) AS cvar
+         FROM ret
+         WHERE ret.ret > - 1
+	     GROUP BY entity_id;
+
+   return (SELECT * FROM t LEFT JOIN t1 ON t.entity_id = t1.entity_id AND t.end_date = t1.end_date AND t.price_date = t1.price_date);
+
+}
+
+
+/*
+ *   Lower Partial Moment
+ *   NOTE: risk free rate is used as Minimal Accepted Rate (MAR) here
+ * 
+ */
+def cal_LPM(ret, risk_free_rate) {
+	
+	t = SELECT *, count(entity_id) AS cnt FROM ret WHERE ret > -1 CONTEXT BY entity_id;
+
+    lpm = SELECT t.entity_id, max(t.end_date) AS end_date,
+                 (sum (rfr.ret - t.ret) \ (t.cnt[0])).pow(1\1) AS lpm1, 
+                 (sum2(rfr.ret - t.ret) \ (t.cnt[0])).pow(1\2) AS lpm2, 
+                 (sum3(rfr.ret - t.ret) \ (t.cnt[0])).pow(1\3) AS lpm3
+          FROM t 
+          INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+          WHERE t.ret < rfr.ret
+          GROUP BY t.entity_id;
+
+    return lpm;
+}
+
+/*
+ *     Downside Devision, Omega Ratio, Sortino Ratio, Kappa Ratio
+ * 
+ *     TODO: Java version of Downside Deviation (LPM2) uses cnt-1 as denominator to calculate mean excess return, which might be wrong
+ *           Java version of Omega could be wrong because Java uses annualized returns and cnt-1 
+ *           Java'version of Kappa could be very wrong
+ *           
+ */
+def cal_omega_sortino_kappa(ret, risk_free_rate) {
+
+    lpm = cal_LPM(ret, risk_free_rate);
+
+    tb = SELECT t.entity_id, 
+                l.lpm2[0] AS ds_dev,
+                (t.ret - rfr.ret ).mean() \ l.lpm1[0] + 1 AS omega,
+                (t.ret - rfr.ret ).mean() \ l.lpm2[0] AS sortino,
+                (t.ret - rfr.ret ).mean() \ l.lpm3[0] AS kappa
+              FROM ret t
+              INNER JOIN lpm l ON t.entity_id = l.entity_id
+              INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+              GROUP BY t.entity_id;
+
+	return tb;
+}
+
+
+/*
+ *   Alpha & Beta
+ *   
+ */
+def cal_alpha_beta(ret, bmk_ret, risk_free) {
+
+	t = SELECT t.entity_id, t.end_date, t.ret, bmk.ret AS ret_bmk
+        FROM ret t
+        INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
+        WHERE t.ret > -1
+          AND bmk.ret > -1;
+
+    beta = SELECT ret.beta(ret_bmk) AS beta FROM t GROUP BY entity_id;
+
+    alpha = SELECT t.entity_id, (t.ret - rfr.ret).mean() - beta.beta[0] * (t.ret_bmk - rfr.ret).mean() AS alpha
+            FROM t 
+            INNER JOIN beta beta ON t.entity_id = beta.entity_id
+            INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+            GROUP BY t.entity_id;
+
+    return ( SELECT * FROM beta AS b INNER JOIN alpha AS a ON a.entity_id = b.entity_id  );
+}
+
+/*
+ *    Winning Ratio, Tracking Error, Information Ratio
+ *    TODO: Information Ratio is way off!
+ *          Not sure how to describe a giant number("inf"), for now 999 is used
+ */
+def cal_benchmark_tracking(ret, bmk_ret) {
+
+	 t0 = SELECT t.entity_id, t.end_date, t.ret, bmk.ret AS ret_bmk, count(entity_id) AS cnt, (t.ret - bmk.ret) AS exc_ret
+          FROM ret t
+          INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
+          WHERE t.ret > -1
+            AND bmk.ret > -1
+          CONTEXT BY t.entity_id;
+
+     t = SELECT entity_id, 
+                exc_ret.bucketCount(0:999, 1) \ cnt[0] AS winrate,
+                exc_ret.std() AS track_error, 
+                exc_ret.mean() / exc_ret.std() AS info
+         FROM t0 GROUP BY entity_id
+
+     return t;
+}
+
+/*
+ *    Sharpe Ratio
+ */
+def cal_sharpe(ret, std_dev, risk_free_rate) {
+
+	sharpe = SELECT t.entity_id, (t.ret - rfr.ret).mean() / std.std_dev[0] AS sharpe
+             FROM ret t
+             INNER JOIN std_dev std ON t.entity_id = std.entity_id
+             INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+             GROUP BY t.entity_id;
+
+    return sharpe;
+}
+
+/*
+ *    Treynor Ratio
+ */
+def cal_treynor(ret, risk_free_rate, beta) {
+
+	t = SELECT *, count(entity_id) AS cnt 
+	   FROM ret t 
+	   INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date 
+	   WHERE t.ret > -1
+	     AND rfr.ret > -1
+	   CONTEXT BY t.entity_id;
+	   
+    treynor = SELECT t.entity_id, ((1 + t.ret).prod().pow(12\iif(t.cnt[0]<12, 12, t.cnt[0])) - (1 + t.rfr_ret).prod().pow(12\iif(t.cnt[0]<12, 12, t.cnt[0]))) / beta.beta[0] AS treynor
+              FROM t
+              INNER JOIN beta AS beta ON t.entity_id = beta.entity_id
+              GROUP BY t.entity_id;
+
+    return treynor;
+}
+
+/*
+ *    Jensen's Alpha
+ *    TODO: the result is slightly off
+ */
+def cal_jensen(ret, bmk_ret, risk_free_rate, beta) {
+
+	jensen = SELECT t.entity_id, t.ret.mean() - rfr.ret.mean() - beta.beta[0] * (bmk.ret.mean() - rfr.ret.mean()) AS jensen
+             FROM ret t
+             INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
+             INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+             INNER JOIN beta beta ON t.entity_id = beta.entity_id
+             GROUP BY t.entity_id;
+               
+    return jensen;
+}
+
+/*
+ *    Calmar Ratio
+ *    TODO: the result is off
+ *
+ */
+def cal_calmar(ret_a){
+
+	calmar = SELECT entity_id, trailing_ret_a \ drawdown AS calmar
+             FROM ret_a;
+
+    return calmar;
+}
+
+/*
+ *     Modigliani Modigliani Measure (M2)
+ *     NOTE: M2 = sharpe * std(benchmark) + risk_free_rate
+ */
+def cal_m2(ret, bmk_ret, risk_free_rate) {
+
+    m2 = SELECT t.entity_id, (t.ret - rfr.ret).mean() / t.ret.std() * bmk.ret.std() + rfr.ret.mean() AS m2
+         FROM ret t
+         INNER JOIN bmk_ret bmk ON t.end_date = bmk.end_date
+         INNER JOIN risk_free_rate rfr ON t.end_date = rfr.end_date
+         GROUP BY t.entity_id;
+
+    return m2;
+}
+
+
+/*
+ *   Monthly Since_inception_date Indicator Calculation
+ *   @param: ret: historical return table
+ *           index_ret: historical benchmark return table
+ *           risk_free: historical risk free rate table
+ *
+ *   @return: indicators table
+ *   
+ *   
+ *   Create  20240904  模仿Java & python代码在Dolphin中实现，具体计算逻辑可能会有不同                          Joey
+ *                     TODO: some datapoints require more data, we need a way to disable calculation for them
+ *
+ */
+def cal_indicators(mutable ret, index_ret, risk_free, freq) {
+
+    if (! freq IN ['d', 'w', 'm', 'q', 's', 'a']) return null;
+
+    // sorting for correct first() and last() value
+    ret.sortBy!(['entity_id', 'price_date'], [1, 1]);
+
+    // 收益、标准差、偏度、峰度、最大回撤、VaR, CVaR
+    rtn = cal_basic_performance(ret);
+
+    // alpha, beta
+    alpha_beta = cal_alpha_beta(ret, index_ret, risk_free);
+
+    // 胜率、跟踪误差、信息比率
+    bmk_tracking = cal_benchmark_tracking(ret, index_ret);
+
+    // 夏普
+    sharpe = cal_sharpe(ret, rtn, risk_free);
+
+    // 特雷诺
+    treynor = cal_treynor(ret, risk_free, alpha_beta);
+
+    // 詹森指数
+    jensen = cal_jensen(ret, index_ret, risk_free, alpha_beta);
+
+    // 卡玛比率
+    calmar = cal_calmar(rtn);
+
+    // 整合后的下行标准差、欧米伽、索提诺、卡帕
+    lpms = cal_omega_sortino_kappa(ret, risk_free);
+
+    // M2
+    m2 = cal_m2(ret, index_ret, risk_free);
+
+    r = SELECT * FROM rtn a1
+                 LEFT JOIN alpha_beta ON a1.entity_id = alpha_beta.entity_id
+                 LEFT JOIN bmk_tracking ON a1.entity_id = bmk_tracking.entity_id
+                 LEFT JOIN sharpe ON a1.entity_id = sharpe.entity_id
+                 LEFT JOIN treynor ON a1.entity_id = treynor.entity_id
+                 LEFT JOIN jensen ON a1.entity_id = jensen.entity_id
+                 LEFT JOIN calmar ON a1.entity_id = calmar.entity_id
+                 LEFT JOIN lpms ON a1.entity_id = lpms.entity_id
+                 LEFT JOIN m2 ON a1.entity_id = m2.entity_id
+
+    // 年化各数据点
+    // GIPS RULE: NO annulization for data less than 1 year
+    plainAnnu = get_annulization_multiple(freq);
+    sqrtAnnu = sqrt(get_annulization_multiple(freq));
+
+    r.addColumn(['ds_dev_a', 'alpha_a', 'sharpe_a', 'sortino_a', 'jensen_a', 'track_error_a', 'info_a', 'm2_a'], [DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE, DOUBLE]);
+
+    UPDATE r 
+      SET ds_dev_a = ds_dev * sqrtAnnu,
+          alpha_a = alpha * plainAnnu,
+          sharpe_a = sharpe * sqrtAnnu,
+          sortino_a = sortino * sqrtAnnu,
+          jensen_a = jensen * plainAnnu,
+          track_error_a = track_error * sqrtAnnu,
+          info_a = info * sqrtAnnu,
+          m2_a = m2 * plainAnnu
+    WHERE price_date.month() - min_date.month() >= 12;
+    
+    return r;
+}
+
+

modules/returnCalculator.dos

@@ -3,57 +3,173 @@ use fundit::fundCalculator
 use fundit::dataPuller
 
 /*
- *  批量计算私募基金收益
- *
- *
- *  cal_hedge_fund_returns("'HF000004KN','HF00018WXG','HF000103EU'", true)
- *
+ *  根据私募基金净值序列计算月收益序列（适合提供给指标运算）
+ * 
+ *  Create:  20240907                                                  Joey
+ *                    TODO: missing pulling data from local
+ *                    TODO: ONLY support month return now
+ *                    
+ *  @param fund_ids: 基金ID STRING VECTOR
+ *  @param isFromMySQL: 净值来源 1 - 远程MySQL、 0 - 本地 DolphinDB
+ *  
  */
-def cal_hedge_fund_returns(fund_ids, isFromMySQL) {
+def cal_hedge_fund_returns(fund_ids, isFromMySQL){
 
+    tb_rets = null;
+    
     // 用于保证老基金也能取到所有历史净值
-    very_old_price_date = 1990.01.01
-
-    // 基金基本信息，包括初始净值
-    tb_fund_info = get_fund_info(fund_ids)
+    very_old_price_date = 1990.01.01;
+
+    if(isFromMySQL){
+    	
+        // 基金基本信息，包括初始净值
+        tb_fund_info = get_fund_info(fund_ids);
+
+        // 基金净值
+        tb_nav = SELECT * FROM get_hedge_fund_nav_by_price_date(fund_ids, very_old_price_date, true);
+
+	    tb_month_end = table(100:0, ['fund_id', 'price_date'], [STRING, DATE]);
+    	// 填充好各基金有效期内所有月份的最后一天
+	    for( f in tb_fund_info )
+    	{
+    		INSERT INTO tb_month_end SELECT fund_id, price_date FROM table(f.fund_id.take(1) AS fund_id).cj(table(temporalSeq(f.inception_date, today(), 'M') AS price_date)) ;
+	    }
     
-    // 基金净值
-    tb_nav = get_hedge_fund_nav_by_price_date(fund_ids, very_old_price_date, isFromMySQL)
+    	UPDATE tb_month_end SET end_date = price_date.month();
     
+	    tb_monthly_nav = SELECT fund_id, monthEnd(price_date).month().last() AS end_date, price_date.last() AS price_date, cumulative_nav.last() AS cumulative_nav
+    	                 FROM tb_nav
+        	             GROUP BY fund_id, monthEnd(price_date);
+
+	    // 完整月末日期的净值序列（包括缺失数据为NULL）
+    	tb_monthly_nav = SELECT me.fund_id, me.end_date, n.price_date, n.cumulative_nav
+        	             FROM tb_month_end me
+            	         LEFT JOIN tb_monthly_nav n ON me.fund_id = n.fund_id AND me.end_date = n.end_date
+                	     ORDER BY me.fund_id, me.end_date;
+
+	    // 补一下成立日的初始净值
+    	// NOTE: DolphinDB 遇见 EXISTS 语句时，似乎主表的 alias 失效，只好用全名
+	    INSERT INTO tb_monthly_nav 
+    		SELECT fund_id, inception_date.month(), inception_date, ifNull(ini_value, 1)
+		    FROM tb_fund_info fi
+    		WHERE NOT EXISTS ( SELECT * FROM tb_monthly_nav n WHERE fund_id = tb_fund_info.fund_id AND n.price_date = tb_fund_info.inception_date);
+
+	    // 算 ratios 之前先把时间顺序排好
+    	tb_monthly_nav.sortBy!(['fund_id', 'end_date', 'price_date'], [1, 1, 1]);
+
+	    // 计算月收益
+    	tb_rets = SELECT fund_id, end_date, price_date, cumulative_nav, cumulative_nav.ratios() - 1 AS ret
+        	      FROM tb_monthly_nav
+            	  CONTEXT BY fund_id;
+
+    }
+
+    // the records without return calculated but do have nav are still useful for some calculations
+    return ( SELECT * FROM tb_rets WHERE cumulative_nav > 0 );
+}
+
+/*
+ *  月末 fund_performance 表计算
+ *  
+ *  @param fund_ids: 逗号分隔的ID
+ *  @param end_date:
+ * 
+ *  Example: cal_fund_performance("'HF000004KN','HF00018WXG','HF000103EU'", '2024-06', true);
+ */
+def cal_fund_performance(fund_ids, month_end) {
+
+    // 获取必要的基金月度净值
+    tb_nav = get_nav_for_hedge_fund_performance(fund_ids, month_end);
+
+    tb_rets = SELECT fund_id, price_date.month() AS end_date, price_date, cumulative_nav,
+                     cumulative_nav \ nav_1m - 1 AS ret_1m,
+                     cumulative_nav \ nav_3m - 1 AS ret_3m,
+                     cumulative_nav \ nav_6m - 1 AS ret_6m,
+                     cumulative_nav \ nav_1y - 1 AS ret_1y,
+                     cumulative_nav \ nav_2y - 1 AS ret_2y,
+                     cumulative_nav \ nav_3y - 1 AS ret_3y,
+                     cumulative_nav \ nav_4y - 1 AS ret_4y,
+                     cumulative_nav \ nav_5y - 1 AS ret_5y,
+                     cumulative_nav \ nav_10y - 1 AS ret_10y,
+                     cumulative_nav \ nav_ytd - 1 AS ret_ytd,
+                     cumulative_nav \ nav_incep - 1 AS ret_incep, inception_date
+              FROM tb_nav;
+
+    // NOTE: this is to keep consistance with MySQL, even it is NOT complied with GIPS standard
+    UPDATE tb_rets SET ret_1m_a = (1 + ret_1m).pow(12\1) - 1, ret_3m_a = (1 + ret_3m).pow(12\3) - 1, ret_6m_a = (1 + ret_6m).pow(12\6) - 1,
+                       ret_1y_a= ret_1y, ret_2y_a = (1 + ret_2y).pow(12\24) - 1, ret_3y_a = (1 + ret_3y).pow(12\36) - 1,
+                       ret_4y_a = (1 + ret_4y).pow(12\48) - 1, ret_5y_a = (1 + ret_5y).pow(12\60) - 1, ret_10y_a = (1 + ret_10y).pow(12\120) - 1,
+                       ret_ytd_a = (1 + ret_ytd).pow(12\int(temporalFormat(end_date, 'MM')))-1,
+                       ret_incep_a = (1 + ret_incep).pow(12\(end_date - inception_date.month())) - 1,
+                       ret_incep_a_all = (1 + ret_incep).pow(12\(end_date - inception_date.month()))- 1,
+                       ret_incep_a_gips = iif(end_date - inception_date.month() < 12, ret_incep,
+                                             (1 + ret_incep).pow(12\(end_date - inception_date.month()))- 1);
+
+    return tb_rets;
+}
+
+
+
+/*
+ *  批量计算公募历史基金月度收益(fund_performance)
+ *  NOTE: 任何数据频率快于或等于月度的净值数据都可以用此函数一次性计算完整历史记录。双月频、季频甚至更低频率的基金只能按月计算
+ *
+ *  cal_mutual_fund_performance("'HF000004KN','HF00018WXG','HF000103EU'", true)
+ *
+ */
+def cal_mutual_fund_performance(fund_ids, isFromMySQL) {
+
+
+/*  找到不必用pivot table 填充数据的办法了
+
     // NOTE: mySQL currently uses calendar day, while the codes below takes business day. it might cause a few different numbers calcuated
     tb_monthly_nav = SELECT fund_id, businessMonthEnd(price_date).month().last() AS end_date, price_date.last() AS price_date, cumulative_nav.last() AS cumulative_nav
                      FROM tb_nav
-                     GROUP BY fund_id, businessMonthEnd(price_date)
+                     GROUP BY fund_id, businessMonthEnd(price_date);
+    
+    // 补一下成立日净值
+    // NOTE: DolphinDB 遇见 EXISTS 语句时，似乎主表的 alias 失效，只好用全名
+    INSERT INTO tb_monthly_nav
+    SELECT businessMonthEnd(inception_date), fund_id, inception_date.month(), inception_date, ifNull(ini_value, 1)
+    FROM tb_fund_info
+    WHERE NOT EXISTS ( SELECT * FROM tb_monthly_nav n WHERE tb_fund_info.fund_id = n.fund_id AND tb_fund_info.inception_date = n.price_date);
     
     // 为了把不数据库里不存在的nav记录填空，不得不先做个pivot；然后才能正确计算ratios (ret_1m)
+    // TODO: much better way is to have a "full list" of dates, then LEFT JOIN nav table, then ffill()
     tb_rets_1m = (SELECT cumulative_nav FROM tb_monthly_nav PIVOT BY end_date, fund_id).ffill!().ratios()-1
     
     // 取被pivot掉的fund_Ids
     v_col_name = tb_rets_1m.columnNames()[1:]
     tb_tmp = tb_rets_1m.unpivot("end_date", v_col_name).rename!("valueType" "value", "fund_id" "ret_1m")
-                         
+*/
+
+    // 计算月收益
+    tb_tmp = cal_hedge_fund_returns(fund_ids, isFromMySQL);
+
     tb_rets = SELECT fund_id, end_date, ret_1m,
                      (1 + ret_1m).mprod(3) - 1 AS ret_3m, (1 + ret_1m).mprod(6) - 1 AS ret_6m, (1 + ret_1m).mprod(12) - 1 AS ret_1y,
                      (1 + ret_1m).mprod(24) - 1 AS ret_2y, (1 + ret_1m).mprod(36) - 1 AS ret_3y, (1 + ret_1m).mprod(48) - 1 AS ret_4y,
                      (1 + ret_1m).mprod(60) - 1 AS ret_5y, (1 + ret_1m).mprod(120) - 1 AS ret_10y
               FROM tb_tmp
-              // WHERE ret_1m IS NOT NULL
-              CONTEXT BY fund_id
-    
+              CONTEXT BY fund_id;
+
+    // NOTE: this is to keep consistance with MySQL, even it is NOT complied with GIPS standard
     UPDATE tb_rets SET ret_1m_a = (1 + ret_1m).pow(12) - 1, ret_3m_a = (1 + ret_3m).pow(4) - 1, ret_6m_a = (1 + ret_6m).pow(2) - 1, ret_1y_a= ret_1y,
                        ret_2y_a = (1 + ret_2y).pow(1\2) - 1, ret_3y_a = (1 + ret_3y).pow(1\3) - 1, ret_4y_a = (1 + ret_4y).pow(1\4) - 1, 
-                       ret_5y_a = (1 + ret_5y).pow(1\5) - 1, ret_10y_a = (1 + ret_10y).pow(1\10) - 1
+                       ret_5y_a = (1 + ret_5y).pow(1\5) - 1, ret_10y_a = (1 + ret_10y).pow(1\10) - 1;
     
     // ytd 不会用上面的CONTEXT BY语句实现
-    tb_ret_ytd = SELECT a.fund_id, a.end_date, a.price_date, a.cumulative_nav, -1 + a.cumulative_nav \ b.cumulative_nav AS ret_ytd, (a.cumulative_nav \ b.cumulative_nav).pow(12\(a.end_date - b.end_date)) - 1 AS ret_ytd_a
-                 FROM tb_monthly_nav a
-                 INNER JOIN tb_monthly_nav b ON a.fund_Id = b.fund_id 
+    tb_ret_ytd = SELECT a.fund_id, a.end_date, a.price_date, a.cumulative_nav, -1 + a.cumulative_nav \ b.cumulative_nav AS ret_ytd,
+                       (a.cumulative_nav \ b.cumulative_nav).pow(12\(a.end_date - b.end_date)) - 1 AS ret_ytd_a
+                 FROM tb_rets a
+                 INNER JOIN tb_rets b ON a.fund_Id = b.fund_id 
                    AND b.end_date = a.price_date.yearEnd().datetimeAdd(-1y).month()
     
     // since inception 不会用上面的CONTEXT BY语句实现
     tb_ret_incep = SELECT a.fund_id, a.end_date, a.price_date,  cumulative_nav, -1 + cumulative_nav \ ini_value AS ret_incep
-                   FROM tb_monthly_nav a
+                   FROM tb_rets a
                    INNER JOIN tb_fund_info fi ON a.fund_id = fi.fund_id
+
     
     UPDATE tb_ret_incep SET ret_incep_a = (1 + ret_incep).pow(12\(end_date - end_date.first())) - 1 CONTEXT BY fund_Id
     UPDATE tb_ret_incep SET ret_incep_a_gips = iif( end_date - end_date.first() < 12, ret_incep, ret_incep_a ), ret_incep_a_all = ret_incep_a CONTEXT BY fund_id