fix: commit latest changes

unoplat-code-confluence/__main__.py

@@ -9,6 +9,7 @@
 from data_models.chapi_unoplat_codebase import UnoplatCodebase
 from downloader.downloader import Downloader
 from dspy_class_summary import CodeConfluenceClassModule
+from dspy_codebase_summary import CodeConfluenceCodebaseModule
 from dspy_function_summary import CodeConfluenceFunctionModule
 from dspy_package_summary import CodeConfluencePackageModule
 from loader import iload_json, iparse_json
@@ -21,30 +22,25 @@
 
 
 def main(iload_json, iparse_json,isummariser,json_configuration_data):
-    #settings = AppSettings()
-    get_codebase_metadata(json_configuration_data,iload_json,iparse_json,isummariser)
+    settings = AppSettings()
+    get_codebase_metadata(json_configuration_data,settings,iload_json,iparse_json,isummariser)
 
 
-def handle_toggle(value):
-    global selected_language
-    selected_language = value
-    logger.info(f"Selected language: {value}")
-
-
-def get_codebase_metadata(json_configuration_data,iload_json,iparse_json,isummariser):
+def get_codebase_metadata(json_configuration_data,settings: AppSettings,iload_json,iparse_json,isummariser):
     # Collect necessary inputs from the user to set up the codebase indexing
     local_workspace_path = json_configuration_data["local_workspace_path"]
     programming_language = json_configuration_data["programming_language"]
     output_path_field = json_configuration_data["output_path"]
     codebase_name_field = json_configuration_data["codebase_name"]
-    github_token = json_configuration_data["repo"]["github_token"]
+    github_token = settings.github_token
     arcguard_cli_repo = json_configuration_data["repo"]["download_url"]
     local_download_directory = json_configuration_data["repo"]["download_directory"]
 
 
     # Button to submit the indexing
     start_parsing(
         local_workspace_path,
+        settings,
         # move this when expanding to new languages
         programming_language,
         output_path_field,
@@ -87,7 +83,7 @@ def ensure_jar_downloaded(github_token,arcguard_cli_repo,local_download_director
 
     return jar_path
 
-def start_parsing(local_workspace_path, programming_language, output_path, codebase_name, github_token, arcguard_cli_repo, local_download_directory, iload_json, iparse_json, isummariser):
+def start_parsing(local_workspace_path, settings, programming_language, output_path, codebase_name, github_token, arcguard_cli_repo, local_download_directory, iload_json, iparse_json, isummariser):
 
     # Log the start of the parsing process
     logger.info("Starting parsing process...")
@@ -125,8 +121,10 @@ def start_parsing(local_workspace_path, programming_language, output_path, codeb
     dspy_class_pipeline_summary : CodeConfluenceClassModule = CodeConfluenceClassModule()
 
     dspy_package_pipeline_summary : CodeConfluencePackageModule = CodeConfluencePackageModule()
+
+    dspy_codebase_pipeline_summary: CodeConfluenceCodebaseModule = CodeConfluenceCodebaseModule()
 
-    codebase_summary = CodebaseSummaryParser(unoplat_codebase,dspy_function_pipeline_summary, dspy_class_pipeline_summary,dspy_package_pipeline_summary)
+    codebase_summary = CodebaseSummaryParser(unoplat_codebase,dspy_function_pipeline_summary, dspy_class_pipeline_summary,dspy_package_pipeline_summary,dspy_codebase_pipeline_summary,settings)
 
     codebase_summary.parse_codebase()
 

unoplat-code-confluence/data_models/dspy/dspy_unoplat_fs_node_subset.py

@@ -15,3 +15,4 @@ class DspyUnoplatNodeSubset(BaseModel):
     imports: Optional[List[Import]] = Field(default_factory=List, alias="Imports", description="This includes class imports which can be used to infer types of fields")
     annotations: Optional[List[DspyUnoplatAnnotationSubset]] = Field(default_factory=list, alias="Annotations")
     functions: List[DspyUnoplatFunctionSubset] = Field(default_factory=list, alias="Functions",exclude=True)
+    content: Optional[str] = Field(default=None, alias="Content",exclude=True)

unoplat-code-confluence/dspy_class_summary.py

@@ -6,21 +6,9 @@
 from data_models.dspy.dspy_unoplat_function_summary import DspyUnoplatFunctionSummary
 from data_models.dspy.dspy_unoplat_node_summary import DspyUnoplatNodeSummary
 
-# ollama_codestral = dspy.OllamaLocal(model='codestral:22b-v0.1-f16')
-# dspy.configure(lm=ollama_codestral)
 
-# ollama_codestral = dspy.OllamaLocal(model='phi3:14b-medium-4k-instruct-f16')
-# dspy.configure(lm=ollama_codestral)
-
-
-ollama_qwen2 = dspy.OllamaLocal(model='qwen2:7b-text-q8_0',max_tokens=1000)
-dspy.configure(lm=ollama_qwen2)
-# ollama_llama_70b = dspy.OllamaLocal(model='llama3:70b-instruct')
-# dspy.configure(lm=ollama_llama_70b)
-
-
-class CodeConfluenceFunctionClassSignature(dspy.Signature):
-    """This signature takes in existing summary of a class and function summary of a class one at a time and returns enhanced final summary"""
+class CodeConfluenceClassSummarySignature(dspy.Signature):
+    """This signature takes in existing summary of a class and function summary of a class one at a time and returns enhanced final summary."""
     class_existing_summary: str = dspy.InputField(default="Summary:",desc="This will contain existing class summary")
     function_summary: str = dspy.InputField(desc="This will contain current function summary based on which existing class summary has to be improved")
     class_metadata: str = dspy.InputField(desc="This will contain current class metadata")
@@ -29,14 +17,14 @@ class CodeConfluenceFunctionClassSignature(dspy.Signature):
 
 class CodeConfluenceClassObjectiveSignature(dspy.Signature):
     """This signature takes in class summary and returns concise objective of the class"""
-    final_class_summary: str = dspy.InputField(desc="This should contain concise detailed implementation summary of the class")
-    class_objective: str = dspy.OutputField(desc="This should contain concise objective of the class based on implementation summary")
+    final_class_summary: str = dspy.InputField(desc="This should contain concise detailed implementation summary of the class or in some cases direct content of the class if it is just a data model object")
+    class_objective: str = dspy.OutputField(desc="This should contain concise objective of the class based on implementation summary in under 2 lines without loosing on any details")
 
 
 class CodeConfluenceClassModule(dspy.Module):
     def __init__(self):
         super().__init__()
-        self.generate_class_summary = dspy.ChainOfThought(CodeConfluenceFunctionClassSignature)
+        self.generate_class_summary = dspy.ChainOfThought(CodeConfluenceClassSummarySignature)
         self.generate_class_objective = dspy.ChainOfThought(CodeConfluenceClassObjectiveSignature)
 
     def forward(self, class_metadata: DspyUnoplatNodeSubset, function_objective_summary: List[DspyUnoplatFunctionSummary]):
@@ -45,9 +33,14 @@ def forward(self, class_metadata: DspyUnoplatNodeSubset, function_objective_summ
         for function_objective in function_objective_summary:
             signature_class_summary = self.generate_class_summary(class_existing_summary=class_summary, function_summary=function_objective.function_summary.objective, class_metadata=str(class_metadata.model_dump_json()))
             class_summary = signature_class_summary.final_class_summary
-
-        class_objective_signature = self.generate_class_objective(final_class_summary = class_summary)
 
+        print("class summary",class_summary)
+
+        if len(function_objective_summary) > 0:
+            class_objective_signature = self.generate_class_objective(final_class_summary = class_summary)
+        else:
+            class_objective_signature = self.generate_class_objective(final_class_summary = class_metadata.content)
+        print("class objective",class_objective_signature.class_objective) 
         dspy_class_summary = DspyUnoplatNodeSummary(NodeName=class_metadata.node_name,NodeObjective=class_objective_signature.class_objective, NodeSummary=class_summary,FunctionsSummary=function_objective_summary)
 
         return dspy.Prediction(answer=dspy_class_summary)

unoplat-code-confluence/dspy_codebase_summary.py

@@ -1,30 +1,42 @@
-
+from typing import List
 import dspy
+from data_models.dspy.dspy_unoplat_package_summary import DspyUnoplatPackageSummary
+
+
 
 class CodeConfluenceCodebaseSignature(dspy.Signature):
-    """This signature takes in existing summary of a codebase and package objective of a package one at a time to keep improving final package summary"""
-    codebase_existing_summary: str = dspy.InputField(default="package existing summary:",desc="This will contain existing package summary")
+    """This signature takes in existing summary of a codebase and package summary of a package one at a time and returns final_codebase_summary as enhanced final summary of codebase"""
+    codebase_existing_summary: str = dspy.InputField(default="codebase existing summary:",desc="This will contain existing codebase summary")
     package_objective: str = dspy.InputField(desc="This will contain current package objective based on which existing codebase summary has to be improved")
-    final_codebase_summary: str = dspy.OutputField(desc="This will contain improved concise codebase summary")
+    final_codebase_summary: str = dspy.OutputField(desc="This will contain final improved concise codebase summary")
 
 
 class CodeConfluenceCodebaseObjectiveSignature(dspy.Signature):
-    """This signature takes in package summary and returns concise objective of the package"""
-    final_package_summary: str = dspy.InputField(desc="This will contain concise detailed implementation summary of the package")
-    package_objective: str = dspy.OutputField(desc="This will contain concise objective of the package based on package summary")
+    """This signature takes in codebase summary and returns codebase_objective as concise objective of the codebase"""
+    final_codebase_summary: str = dspy.InputField(desc="This will contain concise detailed implementation summary of the codebase")
+    codebase_objective: str = dspy.OutputField(desc="This will contain concise objective of the codebase based on detailed codebase summary")
 
-class CodeConfluencePackageModule(dspy.Module):
+class CodeConfluenceCodebaseModule(dspy.Module):
     def __init__(self):
         super().__init__()
-        self.generate_package_summary = dspy.ChainOfThought(CodeConfluenceCodebaseSignature)
-        self.generate_package_objective = dspy.ChainOfThought(CodeConfluenceCodebaseObjectiveSignature)
+        self.generate_codebase_summary = dspy.ChainOfThought(CodeConfluenceCodebaseSignature)
+        self.generate_codebase_objective = dspy.ChainOfThought(CodeConfluenceCodebaseObjectiveSignature)
+
 
-    def forward(self, class_objective_list: List[DspyUnoplatNodeSummary]):
-        package_summary = ""
-        for class_objective in class_objective_list:
-            signature_package_summary: CodeConfluencePackageSignature = self.generate_package_summary(package_existing_summary=package_summary, class_objective=class_objective.node_objective)
-            package_summary = signature_package_summary.final_package_summary
+    def forward(self, package_objective_list: List[DspyUnoplatPackageSummary]):
+        codebase_summary = ""
+        for package_objective in package_objective_list:
+            signature_package_summary: CodeConfluenceCodebaseSignature = self.generate_codebase_summary(codebase_existing_summary=codebase_summary, package_objective=package_objective.package_objective)
+            codebase_summary = signature_package_summary.final_codebase_summary
+            print(codebase_summary)
 
-        class_objective_signature: CodeConfluencePackageObjectiveSignature = self.generate_package_objective(final_package_summary=package_summary)
-        dspy_package_summary = DspyUnoplatPackageNodeSummary(package_objective=class_objective_signature.package_objective,package_summary=package_summary,class_summary=class_objective_list)
-        return dspy_package_summary
+        codebase_objective_signature: CodeConfluenceCodebaseObjectiveSignature = self.generate_codebase_objective(final_codebase_summary=codebase_summary)
+
+
+        return dspy.Prediction(answer=codebase_objective_signature.codebase_objective,summary=signature_package_summary.final_codebase_summary)
+
+
+
+
+
+

unoplat-code-confluence/dspy_function_summary.py

@@ -3,20 +3,6 @@
 from data_models.dspy.dspy_unoplat_fs_function_subset import DspyUnoplatFunctionSubset
 from data_models.dspy.dspy_unoplat_fs_node_subset import DspyUnoplatNodeSubset
 
-# ollama_codestral = dspy.OllamaLocal(model='codestral:22b-v0.1-f16')
-# dspy.configure(lm=ollama_codestral)
-
-# ollama_codestral = dspy.OllamaLocal(model='phi3:14b-medium-4k-instruct-f16')
-# dspy.configure(lm=ollama_codestral)
-
-ollama_qwen2 = dspy.OllamaLocal(model='qwen2:7b-text-q8_0',model_type='text',max_tokens=1000)
-dspy.configure(lm=ollama_qwen2)
-
-
-# ollama_qwen2 = dspy.OllamaLocal(model='qwen2:7b-instruct-q8_0',model_type='text',max_tokens=1000)
-# dspy.configure(lm=ollama_qwen2)
-# ollama_llama_70b = dspy.OllamaLocal(model='llama3:70b-instruct')
-# dspy.configure(lm=ollama_llama_70b)
 
 
 class CodeConfluenceFunctionSummarySignature(dspy.Signature):
@@ -29,7 +15,7 @@ class CodeConfluenceFunctionSummarySignature(dspy.Signature):
 class CodeConfluenceFunctionObjectiveSignature(dspy.Signature):
     """This signature takes in function implementation description and returns objective of the function in a concise and accurate manner."""
     function_implementation: str = dspy.InputField(desc="This will contain concise detailed implementation description of the function")
-    function_objective: str = dspy.OutputField(desc="This will contain concise objective of the function based on implementation summary")
+    function_objective: str = dspy.OutputField(desc="This will contain concise objective of the function based on implementation summary within 3 lines without missing on any details.")
 
 class CodeConfluenceFunctionModule(dspy.Module):
     def __init__(self):
@@ -42,7 +28,9 @@ def forward(self, function_metadata: DspyUnoplatFunctionSubset, class_metadata:
         class_subset = str(class_metadata.model_dump_json())
         function_subset = str(function_metadata.model_dump_json())
         code_confluence_function_summary = self.generate_function_summary( dspy_class_subset = class_subset, dspy_function_subset= function_subset)
+        print("function implementation:",code_confluence_function_summary.function_implementation)
         code_confluence_function_objective = self.generate_function_objective(function_implementation=code_confluence_function_summary.function_implementation)
+        print("function objective:",code_confluence_function_objective.function_objective)
         dspy_function_summary = DspyFunctionSummary(Objective=code_confluence_function_objective.function_objective, ImplementationSummary=code_confluence_function_summary.function_implementation)
         return dspy.Prediction(answer=dspy_function_summary)
 

unoplat-code-confluence/dspy_package_summary.py

@@ -1,25 +1,8 @@
 from typing import List
 import dspy
-from data_models.dspy.dspy_o_function_summary import DspyFunctionSummary
-from data_models.dspy.dspy_unoplat_fs_function_subset import DspyUnoplatFunctionSubset
-from data_models.dspy.dspy_unoplat_fs_node_subset import DspyUnoplatNodeSubset
 from data_models.dspy.dspy_unoplat_node_summary import DspyUnoplatNodeSummary
 from data_models.dspy.dspy_unoplat_package_summary import DspyUnoplatPackageNodeSummary
 
-# ollama_codestral = dspy.OllamaLocal(model='codestral:22b-v0.1-f16')
-# dspy.configure(lm=ollama_codestral)
-
-# ollama_codestral = dspy.OllamaLocal(model='phi3:14b-medium-4k-instruct-f16')
-# dspy.configure(lm=ollama_codestral)
-
-
-ollama_qwen2 = dspy.OllamaLocal(model='qwen2:7b-text-q8_0',model_type='text',max_tokens=1000)
-dspy.configure(lm=ollama_qwen2)
-
-# ollama_qwen2 = dspy.OllamaLocal(model='qwen2:7b-instruct-q8_0',model_type='text',max_tokens=1000)
-# dspy.configure(lm=ollama_qwen2)
-# ollama_llama_70b = dspy.OllamaLocal(model='llama3:70b-instruct')
-# dspy.configure(lm=ollama_llama_70b)
 
 
 class CodeConfluencePackageSignature(dspy.Signature):
@@ -46,6 +29,9 @@ def forward(self, class_objective_list: List[DspyUnoplatNodeSummary]):
         for class_objective in class_objective_list:
             signature_package_summary: CodeConfluencePackageSignature = self.generate_package_summary(package_existing_summary=package_summary, class_objective=class_objective.node_objective)
             package_summary = signature_package_summary.final_package_summary
+            print("current package summary:",package_summary)
+
+        print("final package summary:",package_summary)    
 
         class_objective_signature: CodeConfluencePackageObjectiveSignature = self.generate_package_objective(final_package_summary=package_summary)
         dspy_package_summary = DspyUnoplatPackageNodeSummary(package_objective=class_objective_signature.package_objective,package_summary=package_summary,class_summary=class_objective_list)