merge to one commit

qlib/finco/knowledge.py

@@ -238,10 +238,6 @@ def __init__(self, storages: Union[List[YamlStorage], YamlStorage]):
         storage = self.get_storage(YamlStorage.DEFAULT_NAME)
         if len(storage.documents) == 0:
             docs = self.read_files_in_directory(self.workdir.joinpath(self.name))
-            docs.extend([
-                {"content": "[Success]: XXXX, the results looks reasonable  # Keywords: supervised learning, data"},
-                {"content": "[Fail]: XXXX, it raise memory error due to  YYYYY  "
-                            "# Keywords: supervised learning, data"}])
             self.add(docs)
         self.summarize()
 
@@ -378,20 +374,27 @@ def get_functions_with_docstrings(file_path):
 
 
 class Topic:
-    def __init__(self, name: str, describe: Template):
+    def __init__(self, name: str, system: Template, user: Template):
         self.name = name
-        self.describe = describe
+        self.system_prompt_template = system
+        self.user_prompt_template = user
         self.docs = []
         self.knowledge = None
         self.logger = FinCoLog()
 
-    def summarize(self, docs: list):
-        self.logger.info(f"Summarize Topic \nname: {self.name}\ndescribe: {self.describe.module}")
-        prompt_workflow_selection = self.describe.render(docs=docs)
-        response = APIBackend().build_messages_and_create_chat_completion(user_prompt=prompt_workflow_selection)
+    def summarize(self, practice_knowlege, user_intention, target, diffrence, target_metrics):
+        system_prompt = self.system_prompt_template.render(topic=self.name)
+        user_prompt = self.user_prompt_template.render(
+            experiment_1_info = practice_knowlege[0],
+            experiment_2_info = practice_knowlege[1],
+            user_intention=user_intention,
+            target=target,
+            diffrence=diffrence,
+            target_metrics=target_metrics)
+        response = APIBackend().build_messages_and_create_chat_completion(user_prompt=user_prompt, system_prompt=system_prompt)
 
         self.knowledge = response
-        self.docs = docs
+        self.docs = practice_knowlege
         self.logger.info(f"Summary of {self.name}:\n{self.knowledge}")
 
 
@@ -483,27 +486,48 @@ def query(self, knowledge_type: str = None, content: str = None, n: int = 5):
         # literal search/semantic search
 
         knowledge = self.get_knowledge(knowledge_type=knowledge_type)
-        if len(knowledge) == 0:
+        if len(knowledge) == 0 or knowledge_type == "infrastructure":
             return ""
 
+        if knowledge_type == "practice":
+            knowledge = [line for line in knowledge if line.startswith("practice_knowledge on")]
+
         scores = []
         for k in knowledge:
             scores.append(similarity(str(k), content))
         sorted_indexes = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True)
         similar_n_indexes = sorted_indexes[:n]
-        similar_n_docs = [knowledge[i] for i in similar_n_indexes]
+        similar_n_docs = "\n".join([knowledge[i] for i in similar_n_indexes])
 
-        prompt = Template(
+        user_prompt_template = Template(
 """
-find the most relevant doc with this query: '{{content}}' from docs='{{docs}}'. 
-Just return the most relevant item I provided, no more explain.
-please treat the docs as sentences and always response no less than 5 relevant sentences.
-List all the relevant sentences in number index without any interaction and conversation.
+query: '{{query}}' 
+paragraph:
+{{paragraph}}.
 """
         )
-        prompt_workflow_selection = prompt.render(content=content, docs=similar_n_docs)
+        user_prompt = user_prompt_template.render(query=content, paragraph=similar_n_docs)
+        system_prompt = """
+You are an assistant who find relevant sentences from a long paragraph to fit user's query sentence. Relevant means the sentence might provide userful information to explain user's query sentence. People after reading the relevant sentences might have a better understanding of the query sentence.
+
+Please response no less than ten sentences, if paragraph is not enough, you can return less than ten. Don't pop out irrelevant sentences. Please list the sentences in a number index instead of a whole paragraph.
+
+Example input:
+query: what is the best model for image classification?
+paragraph:
+Image classification is the process of identifying and categorizing objects within an image into different groups or classes.
+Machine learning is a type of artificial intelligence that enables computers to learn and make decisions without being explicitly programmed.
+The solar system is a collection of celestial bodies, including the Sun, planets, moons, and other objects, that orbit around the Sun due to its gravitational pull.
+A car is a wheeled vehicle, typically powered by an engine or electric motor, used for transportation of people and goods.
+ResNet, short for Residual Network, is a type of deep learning architecture designed to improve the accuracy and training speed of neural networks for image recognition tasks.
+
+Example output:
+1. ResNet, short for Residual Network, is a type of deep learning architecture designed to improve the accuracy and training speed of neural networks for image recognition tasks.
+2. Image classification is the process of identifying and categorizing objects within an image into different groups or classes.
+3. Machine learning is a type of artificial intelligence that enables computers to learn and make decisions without being explicitly programmed.
+"""
         response = APIBackend().build_messages_and_create_chat_completion(
-            user_prompt=prompt_workflow_selection, system_prompt="You are an excellent assistant."
+            user_prompt=user_prompt, system_prompt=system_prompt
         )
 
         return response

qlib/finco/llm.py

@@ -26,7 +26,6 @@ def __init__(self, path: Union[Path, str] = DEFAULT_QLIB_DOT_PATH / "llm_conv",
     def _rotate_files(self):
         pairs = []
         for f in self.path.glob("*.json"):
-            print(f)
             m = re.match(r"(\d+).json", f.name)
             if m is not None:
                 n = int(m.group(1))

qlib/finco/prompt_template.yaml

@@ -37,26 +37,19 @@ IdeaTask_system : |-
   Rolling or DDGDA is a kind of data controller which applys custom weight to data in time dimention. So set Data as target module when rolling or DDGDA is used in business level. Never do research both on rolling or DDGDA.
   We often use linear model as default model supervised learning because it trains very fast. If the user didn't plan to achieve very high accuracy, use default model and datahandler is a good choice to save time.
 
-  User will tell you the knowledge type and content in the conversation, if user said "following lists the {practice or finance} knowledge:", you should memorize and understand them then answer "OK" without any other words, finally, user will tell you the research intention, you should answer exactly the same format as the input without any interaction or conversation.
+  User will tell you the type and content of knowledge and the research intention, you should answer exactly the same format as the input without any interaction or conversation.
 
   Example input:
-  Input 1:
   following lists the practice knowledge:
   …
   …
-  Output 1:
-  OK
 
-  Input 2:
   following lists the finance knowledge:
   …
   …
-  Output 2:
-  OK
-  Input 3:
-  Research intention:
-  build an US stock market daily portfolio in quantitative investment and maximize the excess return.
-  Output 3:
+  Research intention: build an US stock market daily portfolio in quantitative investment and maximize the excess return.
+
+  Example output:
   Target: maximize the excess return
   Deliverables: a daily quantitative investment strategy in US stock market. A model will be included in the strategy.
   Thinking directions:
@@ -68,7 +61,11 @@ IdeaTask_system : |-
     Because the user wants to maximize the excess return and more complicated model often extracts more deep pattern from the data. So try a more complicated DNN model to get more excess return than a simple linear model.
 
 IdeaTask_user : |-
-  pass
+  following lists the practice knowledge:
+  {{ practice_knowledge }}
+  following lists the finance knowledge:
+  {{ finance_knowledge }}
+  Research intention: {{ user_intention }}
 
 HighLevelPlanTask_system: |- 
   You are an Quant investment Research and development assistant whose job is to determine high level plans to testify user's research intention.
@@ -1057,13 +1054,29 @@ ImplementActionTask_user : |-
   target component: {{target_component}}
 
 SummarizeTask_system : |-
-  You are an expert in quant domain.
-  Your task is to help user to analysis the output of qlib, your main focus is on the backtesting metrics of 
-  user strategies. Warnings reported during runtime can be ignored if deemed appropriate.
-  your information including the strategy's backtest log and runtime log. 
-  You may receive some scripts of the codes as well, you can use them to analysis the output.
-  At the same time, you can also use your knowledge of the Microsoft/Qlib project and finance to complete your tasks.
-  If there are any abnormal areas in the log or scripts, please also point them out.
+  You are an expert in quant domain. Your task is to help user to analyze the output of two experiments in Qlib, your main focus is on the backtesting metrics of user strategies.
+
+  User has conducted two experiments, which differs only in very small part. 
+  On each experiment, user will give you:
+  1. user's intention why doint these experiments
+  2. The id to differ the experiments
+  3. The yaml config of the experiment
+  4. A small description of the experiment
+  5. the backtest metrics of the experiment
+
+  Finally, user will tell you the targte of doing these experiments, difference between the two experiments and target metrics from the user.
+
+  User will provide a figure path which user has generated some images, please include them in your report.
+
+  You should understand user's intention and target, compare the relevant metrics of the two experiments based on user's intention, give conclusion to the target.
+
+  Please make a table to compare the metrics of two experiments, and make it easy to rean like calculating some increase or highlighting some key metrics.
+
+  You should make summarizations to each experiments, conclusions and recommendations to the further reseach experiments to the user and you should make the report longer.
+
+  Notice: 
+  1. max_drawdown might be presented in negative number or positive number, better max_drawdown (also known lower max_drawdown) means the abstract of it is small, so don't compare each max_drawdown with the number, use the abstract of it instead. This is very important because misunderstanding might cause totally wrong conclusion!!!
+  2. try not to say two experiments performs similar because small progress also means better, even two experiments performs similar, you should still point out who is better.
   
   Example output 1:
   The matrix in log shows that your strategy's max draw down is a bit large, based on your annualized return, 
@@ -1103,8 +1116,19 @@ SummarizeTask_system : |-
   Don't list data user doesn't provide.
 
 SummarizeTask_user : |-
-  Here is my information: '{{information}}'
-  My intention is: {{user_intention}}. Please provide me with a summary and   recommendation based on my intention and the information I have provided. There are some figures which absolute path are: {{figure_path}}, You must display these images in markdown using the appropriate image format.
+  Here is my results on two experiments:
+  experiment 1:
+  {{experiment_1_info}}
+  experiment 2:
+  {{experiment_2_info}}
+  target:
+  {{ target }}
+  difference:
+  {{ difference }}
+  target metrics:
+  {{ target_metrics }}
+  My intention is: {{user_intention}}. 
+  Please provide me with a summary and recommendation based on my intention and the information I have provided. There are some figures which absolute path are: {{figure_path}}, You must display these images in markdown using the appropriate image format.
 
 SummarizeTask_context_system : |-
   Your purpose is to find out the important information offered by user. You can just show the data provided by user in markdown format.
@@ -1131,11 +1155,38 @@ LearnManager_user : |-
   If you have no idea how to optimize the system prompt, you can just return the original system prompt.
   you will adjust {{task}}'s system prompt to:
 
-Topic_IC : |-
-  Summarize the influence of parameters on IC: {{docs}}. (Example response: Max draw-down become larger over time)
-
-Topic_MaxDropDown : |-
-  Summarize the influence of parameters on max dropdown: {{docs}}. (Example response: Max draw-down become larger over time)
-
-Topic_RollingModel : |-
-  What conclusion can you draw from: {{docs}}. Answer questions as concisely as possible. (Example response: rolling model is good at making the Max draw-down smaller.)
+Topic_user : |-
+  experiment 1:
+  {{experiment_1_info}}
+  experiment 2:
+  {{experiment_2_info}}
+  target:
+  {{ target }}
+  difference:
+  {{ difference }}
+  target metrics:
+  {{ target_metrics }}
+  My intention is: {{user_intention}}. 
+
+Topic_system : |-
+  Your job is to summarize the influence of parameters on max dropdown.
+
+  User has conducted two experiments, which differs only in very small part. 
+  On each experiment, user will give you:
+  1. user's intention why doint these experiments
+  2. The id to differ the experiments
+  3. The yaml config of the experiment
+  4. A small description of the experiment
+  5. the backtest metrics of the experiment
+
+  Finally, user will tell you the targte of doing these experiments, difference between the two experiments and target metrics from the user.
+
+  You should compare the metrics of two experiments and give a conclusion on the effection of the difference of the experiments on the topic {{ topic }}.
+
+  Notice: max_drawdown might be presented in negative number or positive number, better max_drawdown means the abstract of it is small, so don't compare each max_drawdown with the number, use the abstract of it instead. This is very important because misunderstanding might cause totally wrong conclusion!!!
+
+  After that, you should give a small explanation to your conclusion.
+
+  Example output format:
+  rolling model is good at making the Max draw-down smaller.
+  explanation: ...