Ci cd build

.github/workflows/build-qa.yml

@@ -54,7 +54,7 @@ jobs:
             GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   publish_docker_container:
-    if: github.event_name == 'release' && startsWith(github.ref, 'refs/tags')
+    if: github.event_name == 'push'
     needs: build
     name: Push Docker Image to Docker Hub
     runs-on: ubuntu-latest
@@ -78,13 +78,23 @@ jobs:
         run: echo ${{ steps.docker_build.outputs.digest }}
 
   publish_pypi_library:
-    if: github.event_name == 'release' && startsWith(github.ref, 'refs/tags')
+    if: github.event_name == 'release'
     needs: build
     name: Publishes pkt_kg to PyPI
     runs-on: ubuntu-latest
     steps:
-      - name: Publish package
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+      - uses: actions/checkout@v2
+        with:
+            fetch-depth: 0
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
+      - name: Install Requirements and Dependencies
+        run: pip install --upgrade pip setuptools wheel
+      - name: Create Package Distribution
+        run: python setup.py sdist bdist_wheel
+      - name: Publish Package tp PyPi
         uses: pypa/gh-action-pypi-publish@master
         with:
           user: __token__

.gitignore

@@ -8,9 +8,9 @@
 __pycache__
 __pycache__/
 **/__pycache__/*
-build/**
+pkt_kg.egg*
+build/*
 dist/*
-pkt_kg.egg-info/*
 
 #### Testing
 .single_run
@@ -20,10 +20,13 @@ coverage.xml
 \.vscode/
 /tests/archive/
 
+#### Logs
+logs/*.log
+**/logs/*.log
+
 #### CI/CD Builds
 builds/temp/*
-builds/GitHub Action Workflow Build Jobs.xlsx
-
+/builds/GitHub Action Workflow Build Jobs.xlsx
 
 #### API Keys/Project Passwords
 /resources/project_keys/*
@@ -63,4 +66,4 @@ scratch*.py
 !/resources/relations_data/README.md
 
 ## Releases
-/releases/*
+/releases/*

Data_Preparation.ipynb

@@ -944,7 +944,7 @@
    "outputs": [],
    "source": [
     "# download data\n",
-    "url = 'https://storage.googleapis.com/pheknowlator/release_v2.0.0/curated_data/genomic_typing_dict.pkl'\n",
+    "url = 'https://storage.googleapis.com/pheknowlator/curated_data/genomic_typing_dict.pkl'\n",
     "if not os.path.exists(unprocessed_data_location + 'genomic_typing_dict.pkl'):\n",
     "    data_downloader(url, unprocessed_data_location)\n",
     "\n",
@@ -2547,7 +2547,7 @@
    "outputs": [],
    "source": [
     "# download data\n",
-    "url='https://storage.googleapis.com/pheknowlator/release_v2.0.0/curated_data/zooma_tissue_cell_mapping_04JAN2020.xlsx'\n",
+    "url='https://storage.googleapis.com/pheknowlator/curated_data/zooma_tissue_cell_mapping_04JAN2020.xlsx'\n",
     "if not os.path.exists(unprocessed_data_location + 'zooma_tissue_cell_mapping_04JAN2020.xlsx'):\n",
     "    data_downloader(url, unprocessed_data_location)\n",
     "    \n",
@@ -3055,7 +3055,7 @@
    "outputs": [],
    "source": [
     "# download data\n",
-    "url='https://storage.googleapis.com/pheknowlator/release_v2.0.0/curated_data/genomic_sequence_ontology_mappings.xlsx'\n",
+    "url='https://storage.googleapis.com/pheknowlator/curated_data/genomic_sequence_ontology_mappings.xlsx'\n",
     "if not os.path.exists(unprocessed_data_location + 'genomic_sequence_ontology_mappings.xlsx'):\n",
     "    data_downloader(url, unprocessed_data_location)\n",
     "\n",

Dockerfile

@@ -1,5 +1,5 @@
 #!/usr/local/bin/docker
-# -*- version: 19.03.8, build afacb8b -*-
+# -*- version: 20.10.2 -*-
 
 ############################################
 ## MULTI-STAGE CONTAINER CONFIGURATION ##
@@ -19,25 +19,44 @@ RUN wget -O- https://apt.corretto.aws/corretto.key | apt-key add - && \
 ## PHEKNOWLATOR (PKT_KG) PROJECT SETTINGS ##
 # create needed project directories
 WORKDIR /PheKnowLator
-# I THINK WE NEED LINE 23 -- which I had to delete to get it to run past this point
-#RUN mkdir /PheKnowLator
-RUN mkdir /PheKnowLator/resources
-RUN mkdir /PheKnowLator/resources/edge_data
-RUN mkdir /PheKnowLator/resources/knowledge_graphs
-RUN mkdir /PheKnowLator/resources/ontologies
-
-# copy pkt_kg scripts
-COPY pkt_kg /PheKnowLator/pkt_kg
+RUN mkdir -p /PheKnowLator
+RUN mkdir -p /PheKnowLator/resources
+RUN mkdir -p /PheKnowLator/resources/construction_approach
+RUN mkdir -p /PheKnowLator/resources/edge_data
+RUN mkdir -p /PheKnowLator/resources/knowledge_graphs
+RUN mkdir -p /PheKnowLator/resources/node_data
+RUN mkdir -p /PheKnowLator/resources/ontologies
+RUN mkdir -p /PheKnowLator/resources/processed_data
+RUN mkdir -p /PheKnowLator/resources/relations_data
 
 # copy scripts/files needed to run pkt_kg
+COPY pkt_kg /PheKnowLator/pkt_kg
 COPY Main.py /PheKnowLator
 COPY setup.py /PheKnowLator
 COPY README.rst /PheKnowLator
 COPY resources /PheKnowLator/resources
 
+# download and copy needed data
+RUN curl -O https://storage.googleapis.com/pheknowlator/current_build/dependencies/edge_source_list.txt && mv edge_source_list.txt resources/
+RUN curl -O https://storage.googleapis.com/pheknowlator/current_build/dependencies/ontology_source_list.txt && mv ontology_source_list.txt resources/
+RUN curl -O https://storage.googleapis.com/pheknowlator/current_build/dependencies/resource_info.txt && mv resource_info.txt resources/
+RUN curl -O https://storage.googleapis.com/pheknowlator/current_build/dependencies/subclass_construction_map.pkl && mv subclass_construction_map.pkl resources/construction_approach/
+RUN curl -O https://storage.googleapis.com/pheknowlator/current_build/dependencies/PheKnowLator_MergedOntologies.owl && mv PheKnowLator_MergedOntologies.owl resources/knowledge_graphs/
+RUN curl -O https://storage.googleapis.com/pheknowlator/current_build/dependencies/node_metadata_dict.pkl && mv node_metadata_dict.pkl resources/node_data/
+RUN curl -O https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/DISEASE_MONDO_MAP.txt && mv DISEASE_MONDO_MAP.txt resources/processed_data/
+RUN curl -O https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/ENSEMBL_GENE_ENTREZ_GENE_MAP.txt && mv ENSEMBL_GENE_ENTREZ_GENE_MAP.txt resources/processed_data/
+RUN curl -O https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/ENTREZ_GENE_PRO_ONTOLOGY_MAP.txt && mv ENTREZ_GENE_PRO_ONTOLOGY_MAP.txt resources/processed_data/
+RUN curl -O https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/GENE_SYMBOL_ENSEMBL_TRANSCRIPT_MAP.txt && mv GENE_SYMBOL_ENSEMBL_TRANSCRIPT_MAP.txt resources/processed_data/
+RUN curl -O https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/HPA_GTEx_TISSUE_CELL_MAP.txt && mv HPA_GTEx_TISSUE_CELL_MAP.txt resources/processed_data/
+RUN curl -O https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/MESH_CHEBI_MAP.txt && mv MESH_CHEBI_MAP.txt resources/processed_data/
+RUN curl -O https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/PHENOTYPE_HPO_MAP.txt && mv PHENOTYPE_HPO_MAP.txt resources/processed_data/
+RUN curl -O https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/STRING_PRO_ONTOLOGY_MAP.txt && mv STRING_PRO_ONTOLOGY_MAP.txt resources/processed_data/
+RUN curl -O https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt && mv UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt resources/processed_data/
+RUN curl -O https://storage.googleapis.com/pheknowlator/current_build/dependencies/INVERSE_RELATIONS.txt && mv INVERSE_RELATIONS.txt resources/relations_data/
+RUN curl -O https://storage.googleapis.com/pheknowlator/current_build/dependencies/RELATIONS_LABELS.txt && mv RELATIONS_LABELS.txt resources/relations_data/
+
 # install needed python libraries
 RUN pip install --upgrade pip setuptools
-# BILL -- CAN WE REMOVE LINE  41 if we had to add line 21 to get the container to run
 WORKDIR /PheKnowLator
 RUN pip install .
 

MANIFEST.in

@@ -0,0 +1 @@
+include pkt_kg/libs/owltools

Main.py

@@ -21,10 +21,9 @@ def main():
     parser.add_argument('-a', '--app', help='construction approach to use (i.e. instance or subclass)', required=True)
     parser.add_argument('-t', '--res', help='name/path to text file containing resource_info', required=True)
     parser.add_argument('-b', '--kg', help='build type: "partial", "full", or "post-closure"', required=True)
-    parser.add_argument('-n', '--nde', help='yes/no - non-ontology node metadata directory location', required=True)
     parser.add_argument('-r', '--rel', help='yes/no - adding inverse relations to knowledge graph', required=True)
     parser.add_argument('-s', '--owl', help='yes/no - removing OWL Semantics from knowledge graph', required=True)
-    parser.add_argument('-m', '--kgm', help='yes/no - adding node metadata to knowledge graph', required=True)
+    parser.add_argument('-m', '--nde', help='yes/no - adding node metadata to knowledge graph', required=True)
     parser.add_argument('-o', '--out', help='name/path to directory where to write knowledge graph', required=True)
 
     args = parser.parse_args()
@@ -40,44 +39,44 @@ def main():
     # see the 'Data_Preparation.ipynb' and 'Ontology_Cleaning.ipynb' file for examples and guidelines
 
     # STEP 3: DOWNLOAD ONTOLOGIES
-    print('\n' + '=' * 33 + '\nDOWNLOADING DATA: ONTOLOGY DATA\n' + '=' * 33 + '\n')
+    print('\n' + '=' * 33 + '\nPKT: DOWNLOADING DATA: ONTOLOGY DATA\n' + '=' * 33 + '\n')
     start = time.time()
     ont = OntData(data_path=args.onts, resource_data=args.res)
     # ont = OntData(data_path='resources/ontology_source_list.txt', resource_data='resources/resource_info.txt')
     ont.downloads_data_from_url()
     end = time.time()
     timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    print('\nTOTAL SECONDS TO DOWNLOAD ONTOLOGIES: {} @ {}'.format(end - start, timestamp))
+    print('\nPKT: TOTAL SECONDS TO DOWNLOAD ONTOLOGIES: {} @ {}'.format(end - start, timestamp))
 
     # STEP 4: DOWNLOAD EDGE DATA SOURCES
-    print('\n' + '=' * 33 + '\nDOWNLOADING DATA: CLASS DATA\n' + '=' * 33 + '\n')
+    print('\n' + '=' * 33 + '\nPKT: DOWNLOADING DATA: CLASS DATA\n' + '=' * 33 + '\n')
     start = time.time()
     ent = LinkedData(data_path=args.edg, resource_data=args.res)
     # ent = LinkedData(data_path='resources/edge_source_list.txt', resource_data='resources/resource_info.txt')
     ent.downloads_data_from_url()
     end = time.time()
     timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    print('\nTOTAL SECONDS TO DOWNLOAD NON-ONTOLOGY DATA: {} @ {}'.format(end - start, timestamp))
+    print('\nPKT: TOTAL SECONDS TO DOWNLOAD NON-ONTOLOGY DATA: {} @ {}'.format(end - start, timestamp))
 
     #####################
     # CREATE EDGE LISTS #
     #####################
 
-    print('\n' + '=' * 33 + '\nPROCESSING EDGE DATA\n' + '=' * 33 + '\n')
+    print('\n' + '=' * 33 + '\nPKT: PROCESSING EDGE DATA\n' + '=' * 33 + '\n')
     start = time.time()
     combined_edges = dict(ent.data_files, **ont.data_files)
     master_edges = CreatesEdgeList(data_files=combined_edges, source_file=args.res)
     # master_edges = CreatesEdgeList(data_files=combined_edges, source_file='resources/resource_info.txt')
     master_edges.creates_knowledge_graph_edges()
     end = time.time()
     timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    print('\nTOTAL SECONDS TO BUILD THE MASTER EDGE LIST: {} @ {}'.format(end - start, timestamp))
+    print('\nPKT: TOTAL SECONDS TO BUILD THE MASTER EDGE LIST: {} @ {}'.format(end - start, timestamp))
 
     #########################
     # BUILD KNOWLEDGE GRAPH #
     #########################
 
-    print('\n' + '=' * 33 + '\nBUILDING KNOWLEDGE GRAPH\n' + '=' * 33 + '\n')
+    print('\n' + '=' * 33 + '\nPKT: BUILDING KNOWLEDGE GRAPH\n' + '=' * 33 + '\n')
     start = time.time()
 
     if args.kg == 'partial':
@@ -102,7 +101,7 @@ def main():
 
     end = time.time()
     timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    print('\nTOTAL SECONDS TO CONSTRUCT A KG: {} @ {}'.format(end - start, timestamp))
+    print('\nPKT: TOTAL SECONDS TO CONSTRUCT A KG: {} @ {}'.format(end - start, timestamp))
 
 
 if __name__ == '__main__':

Ontology_Cleaning.ipynb

@@ -28,7 +28,7 @@
     "**Dependencies:**   \n",
     "- <u>Scripts</u>: This notebook utilizes several helper functions, which are stored in [utility scripts](https://github.com/callahantiff/PheKnowLator/blob/master/pkt_kg/utils) and the main code to perform the error checks in the  [`builds/temp/ontology_cleaning.py`](https://github.com/callahantiff/PheKnowLator/blob/master/builds/ontology_cleaning.py) script.\n",
     "- <u>Software</u>:[`OWLTools`](https://github.com/owlcollab/owltools)  \n",
-    "- <u>Data</u>: [`Merged_gene_rna_protein_identifiers.pkl`](https://storage.googleapis.com/pheknowlator/release_v2.0.0/current_build/data/processed_data/Merged_gene_rna_protein_identifiers.pkl), which is automatically downloaded to the `./resources/ontologies` directory     \n",
+    "- <u>Data</u>: [`Merged_gene_rna_protein_identifiers.pkl`](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/Merged_gene_rna_protein_identifiers.pkl), which is automatically downloaded to the `./resources/ontologies` directory     \n",
     "\n",
     "<br>\n",
     "\n",
@@ -194,6 +194,7 @@
     "import datetime\n",
     "import glob\n",
     "import pickle\n",
+    "import shutil\n",
     "\n",
     "from rdflib import Graph\n",
     "from tqdm import tqdm\n",
@@ -299,7 +300,10 @@
     "ont_data = OntologyCleaner('', '', '', write_location)\n",
     "\n",
     "# updating ontology info dictionary\n",
-    "ont_data.ontology_info = {k.split('/')[-1]: {} for k, v in ont_data.ontology_info.items()}"
+    "ont_data.ontology_info = {k.split('/')[-1]: {} for k, v in ont_data.ontology_info.items()}\n",
+    "\n",
+    "# set owl tools location\n",
+    "ont_data.owltools_location = './pkt_kg/libs/owltools'"
    ]
   },
   {
@@ -426,15 +430,6 @@
     "ontology_file_formatter(knowledge_graphs_location, '/' + ont_data.ont_file_location, ont_data.owltools_location)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "ontology_file_formatter(knowledge_graphs_location, '/' + ont_data.ont_file_location, ont_data.owltools_location)"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -480,6 +475,11 @@
     "            else: o.write('\\t\\t- {}\\n'.format(x['Normalized - Duplicates']))\n",
     "            o.write('\\t\\t- Other Classes that May Need Normalization: {}\\n'.format(x['Normalized - NonOnt']))\n",
     "            o.write('\\t\\t- Normalized HGNC IDs: {}\\n'.format(x['Normalized - Gene IDs']))\n",
+    "            o.write('\\t- Deprecated Ontology HGNC Identifiers Needing Alignment:\\n')\n",
+    "            if x['Normalized - Dep'] != 'None':\n",
+    "                for i in x['Normalized - Dep']: o.write('\\t\\t- {}\\n'.format(i))\n",
+    "            else: o.write('\\t\\t- {}\\n'.format(x['Normalized - Dep']))\n",
+    "    \n",
     "o.close()"
    ]
   },
@@ -498,7 +498,11 @@
    "source": [
     "# remove temp file in resources/ontologies\n",
     "os.remove(write_location + '/' + ont_data.ont_file_location)\n",
-    "os.remove(write_location + '/Merged_gene_rna_protein_identifiers.pkl')"
+    "os.remove(write_location + '/Merged_gene_rna_protein_identifiers.pkl')\n",
+    "\n",
+    "# # remove logs directory\n",
+    "# logs = glob.glob('*/logs/*.log')\n",
+    "# shutil.rmtree('/'.join(logs[0].split('/')[:-1]))"
    ]
   },
   {

README.rst

@@ -1,7 +1,7 @@
-|logo|
+|logo| 
 
 
-|github_action| |mypy|
+|github_action| |mypy|  
 
 |sonar_quality| |code_climate_maintainability| |codacy| |code_climate_coverage| |coveralls|
 
@@ -325,11 +325,12 @@ Citing this Work
      url          = {https://doi.org/10.5281/zenodo.3401437}}
 
 
-
-
 .. |logo| image:: https://user-images.githubusercontent.com/8030363/106306246-01df9100-621b-11eb-81c3-d1f2c2e124a6.png
    :target: https://github.com/callahantiff/PheKnowLator
 
+.. |logo| image:: https://user-images.githubusercontent.com/8030363/106306246-01df9100-621b-11eb-81c3-d1f2c2e124a6.png
+   :target: https://github.com/callahantiff/PheKnowLator  
+
 .. |ABRA| image:: https://img.shields.io/badge/ReproducibleResearch-AbraCollaboratory-magenta.svg
    :target: https://github.com/callahantiff/Abra-Collaboratory