init

lvxhnat · Feb 4, 2023 · 6851092 · 6851092
commit 6851092
Show file tree

Hide file tree

Showing 23 changed files with 1,866 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,7 @@
+.env
+
+dist/
+__pycache__/
+.vscode/
+
+*.csv
diff --git a/LICENSE b/LICENSE
diff --git a/README.md b/README.md
@@ -0,0 +1,103 @@
+# pyetfdb_scraper
+```pyetfdb_scraper``` is a Python library for extracting ETF data directly from [ETFDB](https://etfdb.com/), a website providing one of the largest ETF Databases containing ETFs from a vast range of asset classes, industries, issuers, and investment styles.
+
+## Quick Start 
+Install with ```pip``` as a package pip. See the pip package here https://pypi.org/project/pyetfdb-scraper/.
+
+```
+pip install pyetfdb_scraper
+```
+
+```python
+from pyetfdb_scraper import etf
+```
+
+## Example Usage
+
+```python
+from pyetfdb_scraper.etf import ETF, list_etfs
+# returns list of available ETFs.
+etfs = list_etfs()
+# load etf
+vwo = ETF('VWO')
+# Get basic ETF information
+print(vwo.info)
+>>> {
+     '52 Week Hi': '$55.78',
+     '52 Week Lo': '$47.65',
+     'AUM': '$80,421.8 M',
+     'Asset Class': 'Equity',
+     'Asset Class Size': 'Large-Cap',
+     'Asset Class Style': 'Blend',
+     'Brand': 'https://etfdb.com/issuer/vanguard/',
+     'Category': 'Size and Style',
+     'Category:': 'Emerging Markets Equities',
+     'Change:': '$0.25 (-0.0%)',
+     'ETF Home Page': 'https://advisors.vanguard.com/investments/products/bnd/vanguard-total-bond-market-etf',
+     'Expense Ratio': '0.10%',
+     'Focus': 'Total Market',
+     'Inception': 'Mar 04, 2005',
+     'Index Tracked': 'https://etfdb.com/index/ftse-custom-emerging-markets-all-cap-china-a-inclusion-net-tax-us-ric-index/',
+     'Issuer': 'https://etfdb.com/issuer/vanguard/',
+     'Last Updated:': 'Dec 09, 2021',
+     'Niche': 'Broad-based',
+     'P/E Ratio': '7.00',
+     'Price:': '$50.14',
+     'Region (General)': 'Emerging Markets',
+     'Region (Specific)': 'Broad',
+     'Segment': 'Equity: Emerging Markets  -  Total Market',
+     'Shares': '1,603.3 M',
+     'Strategy': 'Vanilla',
+     'Structure': 'ETF',
+     'Weighting Scheme': 'Market Cap'
+ }
+
+print(vwo.technicals)
+
+>>> {
+     '20 Day MA': '$50.45',
+     '60 Day MA': '$50.74',
+     'Average Spread ($)': '1.00',
+     'Average Spread (%)': '1.00',
+     'Lower Bollinger (10 Day)': '$48.64',
+     'Lower Bollinger (20 Day)': '$48.33',
+     'Lower Bollinger (30 Day)': '$48.81',
+     'MACD 100 Period': '-0.74',
+     'MACD 15 Period': '0.20',
+     'Maximum Premium Discount (%)': '0.82',
+     'Median Premium Discount (%)': '0.27',
+     'RSI 10 Day': '49',
+     'RSI 20 Day': '47',
+     'RSI 30 Day': '47',
+     'Resistance Level 1': 'n/a',
+     'Resistance Level 2': '$50.53',
+     'Stochastic Oscillator %D (1 Day)': '53.54',
+     'Stochastic Oscillator %D (5 Day)': '73.08',
+     'Stochastic Oscillator %K (1 Day)': '55.09',
+     'Stochastic Oscillator %K (5 Day)': '57.68',
+     'Support Level 1': 'n/a',
+     'Support Level 2': '$49.86',
+     'Tracking Difference Max Downside (%)': '-0.87',
+     'Tracking Difference Max Upside (%)': '0.16',
+     'Tracking Difference Median (%)': '-0.36',
+     'Ultimate Oscillator': '47',
+     'Upper Bollinger (10 Day)': '$50.47',
+     'Upper Bollinger (20 Day)': '$52.61',
+     'Upper Bollinger (30 Day)': '$52.50',
+     'Williams % Range 10 Day': '19.32',
+     'Williams % Range 20 Day': '59.31'
+}
+```
+## Help Needed!
+I am working full-time, and as such don't have much time to constantly push commits or updates. I will appreciate if some help can be provided, such as: 
+* Unit tests for the current code
+* ETF Category has yet to be updated
+
+## Disclaimer 
+This package is built with some reference to the existing [pyetf](https://github.com/JakubPluta/pyetf) package creted by Jakub Pluta, which has since not been actively maintained.
+
+## Contributing
+Pull requests are welcome.
+
+## License
+MIT License
diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,54 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.darker]
+revision = "94f36d8d0cde18b5f2b6a2a4b009002e08bbd959"
+isort = true
+log_level = "INFO"
+line-length = 79
+
+[tool.black]
+line-length = 79
+
+[tool.isort]
+line_length = 79
+profile = "black"
+known_first_party = [
+    "common", "env", "backend", "interface", "engine", "cli", "experimental"
+]
+default_section = "THIRDPARTY"
+
+[tool.pylint]
+[tool.pylint.master]
+unsafe-load-any-extension = false
+
+[tool.pylint.messages_control]
+disable = [
+    "line-too-long", "no-else-return"
+]
+enable = ["useless-suppression"]
+
+[tool.pylint.refactoring]
+max-nested-blocks = 5
+
+[tool.pylint.format]
+max-line-length = 79
+indent-after-paren= 4
+indent-string = "    "
+
+[tool.pylint.miscellaneous]
+notes = [
+    "TODO",
+    "FIXME",
+]
+
+[tool.mypy]
+disallow_untyped_defs = true
+disallow_any_unimported = true
+no_implicit_optional = true
+check_untyped_defs = true
+warn_return_any = true
+show_error_codes = true
+warn_unused_ignores = true
+ignore_missing_imports = true
diff --git a/setup.py b/setup.py
@@ -0,0 +1,23 @@
+from setuptools import setup
+from setuptools import find_packages
+
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
+setup(
+    name="pyetfdb_scraper",
+    version="0.2.0",
+    author="Yi Kuang",
+    author_email="[email protected]",
+    description="Scrape ETFs from ETFDB",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/lvxhnat/pyetf-scraper",
+    package_dir={"": "src"},
+    packages=find_packages("src", exclude=["*tests"]),
+    python_requires=">=3.7",
+    install_requires=[
+        "requests",
+        "bs4",
+    ],
+)
diff --git a/src/pyetfdb_scraper.egg-info/PKG-INFO b/src/pyetfdb_scraper.egg-info/PKG-INFO
@@ -0,0 +1,113 @@
+Metadata-Version: 2.1
+Name: pyetfdb-scraper
+Version: 0.1.0
+Summary: Scrape ETFs from ETFDB
+Home-page: https://github.com/lvxhnat/pyetf-scraper/
+Author: Yi Kuang
+Author-email: [email protected]
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+
+# pyetfdb_scraper
+```pyetfdb_scraper``` is a Python library for extracting ETF data directly from [ETFDB](https://etfdb.com/), a website providing one of the largest ETF Databases containing ETFs from a vast range of asset classes, industries, issuers, and investment styles.
+
+## Quick Start 
+Install with ```pip``` as a package pip
+
+```
+pip install pyetfdb_scraper
+```
+
+```python
+from pyetf import etfdb
+```
+
+## Example Usage
+
+```python
+from pyetf import etfdb
+# returns list of available ETFs.
+etfs = etfdb.list_etfs()
+# load etf
+vwo = etfdb.load_etf('VWO')
+# Get basic ETF information
+print(vwo.info)
+>>> {
+     '52 Week Hi': '$55.78',
+     '52 Week Lo': '$47.65',
+     'AUM': '$80,421.8 M',
+     'Asset Class': 'Equity',
+     'Asset Class Size': 'Large-Cap',
+     'Asset Class Style': 'Blend',
+     'Brand': 'https://etfdb.com/issuer/vanguard/',
+     'Category': 'Size and Style',
+     'Category:': 'Emerging Markets Equities',
+     'Change:': '$0.25 (-0.0%)',
+     'ETF Home Page': 'https://advisors.vanguard.com/investments/products/bnd/vanguard-total-bond-market-etf',
+     'Expense Ratio': '0.10%',
+     'Focus': 'Total Market',
+     'Inception': 'Mar 04, 2005',
+     'Index Tracked': 'https://etfdb.com/index/ftse-custom-emerging-markets-all-cap-china-a-inclusion-net-tax-us-ric-index/',
+     'Issuer': 'https://etfdb.com/issuer/vanguard/',
+     'Last Updated:': 'Dec 09, 2021',
+     'Niche': 'Broad-based',
+     'P/E Ratio': '7.00',
+     'Price:': '$50.14',
+     'Region (General)': 'Emerging Markets',
+     'Region (Specific)': 'Broad',
+     'Segment': 'Equity: Emerging Markets  -  Total Market',
+     'Shares': '1,603.3 M',
+     'Strategy': 'Vanilla',
+     'Structure': 'ETF',
+     'Weighting Scheme': 'Market Cap'
+ }
+
+print(vwo.technicals)
+
+>>> {
+     '20 Day MA': '$50.45',
+     '60 Day MA': '$50.74',
+     'Average Spread ($)': '1.00',
+     'Average Spread (%)': '1.00',
+     'Lower Bollinger (10 Day)': '$48.64',
+     'Lower Bollinger (20 Day)': '$48.33',
+     'Lower Bollinger (30 Day)': '$48.81',
+     'MACD 100 Period': '-0.74',
+     'MACD 15 Period': '0.20',
+     'Maximum Premium Discount (%)': '0.82',
+     'Median Premium Discount (%)': '0.27',
+     'RSI 10 Day': '49',
+     'RSI 20 Day': '47',
+     'RSI 30 Day': '47',
+     'Resistance Level 1': 'n/a',
+     'Resistance Level 2': '$50.53',
+     'Stochastic Oscillator %D (1 Day)': '53.54',
+     'Stochastic Oscillator %D (5 Day)': '73.08',
+     'Stochastic Oscillator %K (1 Day)': '55.09',
+     'Stochastic Oscillator %K (5 Day)': '57.68',
+     'Support Level 1': 'n/a',
+     'Support Level 2': '$49.86',
+     'Tracking Difference Max Downside (%)': '-0.87',
+     'Tracking Difference Max Upside (%)': '0.16',
+     'Tracking Difference Median (%)': '-0.36',
+     'Ultimate Oscillator': '47',
+     'Upper Bollinger (10 Day)': '$50.47',
+     'Upper Bollinger (20 Day)': '$52.61',
+     'Upper Bollinger (30 Day)': '$52.50',
+     'Williams % Range 10 Day': '19.32',
+     'Williams % Range 20 Day': '59.31'
+}
+```
+## Help Needed!
+I am working full-time, and as such don't have much time to constantly push commits or updates. I will appreciate if some help can be provided, such as: 
+* Unit tests for the current code
+* ETF Category has yet to be updated
+
+## Disclaimer 
+This package is built with some reference to the existing [pyetf](https://github.com/JakubPluta/pyetf) package creted by Jakub Pluta, which has since not been actively maintained.
+
+## Contributing
+Pull requests are welcome.
+
+## License
+MIT License
diff --git a/src/pyetfdb_scraper.egg-info/SOURCES.txt b/src/pyetfdb_scraper.egg-info/SOURCES.txt
@@ -0,0 +1,21 @@
+README.md
+pyproject.toml
+setup.py
+src/pyetfdb_scraper/__init__.py
+src/pyetfdb_scraper/client.py
+src/pyetfdb_scraper/etf.py
+src/pyetfdb_scraper/utils.py
+src/pyetfdb_scraper.egg-info/PKG-INFO
+src/pyetfdb_scraper.egg-info/SOURCES.txt
+src/pyetfdb_scraper.egg-info/dependency_links.txt
+src/pyetfdb_scraper.egg-info/requires.txt
+src/pyetfdb_scraper.egg-info/top_level.txt
+src/pyetfdb_scraper/tabs/__init__.py
+src/pyetfdb_scraper/tabs/dividend.py
+src/pyetfdb_scraper/tabs/expense.py
+src/pyetfdb_scraper/tabs/holding_analysis.py
+src/pyetfdb_scraper/tabs/holdings.py
+src/pyetfdb_scraper/tabs/info.py
+src/pyetfdb_scraper/tabs/performance.py
+src/pyetfdb_scraper/tabs/realtime_ratings.py
+src/pyetfdb_scraper/tabs/technicals.py
diff --git a/src/pyetfdb_scraper.egg-info/dependency_links.txt b/src/pyetfdb_scraper.egg-info/dependency_links.txt
@@ -0,0 +1 @@
+
diff --git a/src/pyetfdb_scraper.egg-info/requires.txt b/src/pyetfdb_scraper.egg-info/requires.txt
@@ -0,0 +1,2 @@
+requests
+bs4
diff --git a/src/pyetfdb_scraper.egg-info/top_level.txt b/src/pyetfdb_scraper.egg-info/top_level.txt
@@ -0,0 +1 @@
+pyetfdb_scraper
diff --git a/src/pyetfdb_scraper/__init__.py b/src/pyetfdb_scraper/__init__.py
diff --git a/src/pyetfdb_scraper/data/etfdb.json b/src/pyetfdb_scraper/data/etfdb.json