update requirements and add pre-commit logic

.github/workflows/python-lint.yml

@@ -1,40 +0,0 @@
-name: Python Lint
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.12'
-          
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install black isort
-          
-      - name: Run Black
-        run: black .
-        
-      - name: Run isort
-        run: isort .
-        
-      - name: Commit changes
-        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-        run: |
-          git config --local user.email "github-actions[bot]@users.noreply.github.com"
-          git config --local user.name "github-actions[bot]"
-          git add .
-          git diff --quiet && git diff --staged --quiet || git commit -m "Apply automatic code formatting"
-          git push

.pre-commit-config.yaml

@@ -0,0 +1,11 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+-   repo: https://github.com/psf/black
+    rev: 22.10.0
+    hooks:
+    -   id: black

requirements.txt

@@ -5,18 +5,22 @@ annotated-types==0.6.0
 antlr4-python3-runtime==4.9.3
 anyio==4.3.0
 appnope==0.1.4
+assemblyai==0.36.0
 asttokens==2.4.1
 attrs==23.2.0
 backoff==2.2.1
 bcrypt==4.0.1
 beautifulsoup4==4.12.3
+black==24.10.0
 boto3==1.34.103
 boto3-stubs==1.34.150
 botocore==1.34.103
 botocore-stubs==1.34.150
+build==1.2.2.post1
 cachetools==5.3.3
 certifi==2024.2.2
 cffi==1.17.0
+cfgv==3.4.0
 chardet==5.2.0
 charset-normalizer==3.3.2
 click==8.1.7
@@ -32,8 +36,10 @@ decorator==5.1.1
 deepdiff==7.0.1
 Deprecated==1.2.14
 dill==0.3.8
+distlib==0.3.9
 distro==1.9.0
 dnspython==2.6.1
+docutils==0.21.2
 ecdsa==0.19.0
 effdet==0.4.1
 email_validator==2.1.1
@@ -63,11 +69,15 @@ httptools==0.6.1
 httpx==0.27.0
 huggingface-hub==0.24.5
 humanfriendly==10.0
+identify==2.6.3
 idna==3.7
 iniconfig==2.0.0
 iopath==0.1.10
 ipykernel==6.29.5
 ipython==8.26.0
+jaraco.classes==3.4.0
+jaraco.context==6.0.1
+jaraco.functools==4.1.0
 jedi==0.19.1
 Jinja2==3.1.3
 jiter==0.5.0
@@ -78,6 +88,7 @@ jsonpath-python==1.0.6
 jsonpointer==3.0.0
 jupyter_client==8.6.2
 jupyter_core==5.7.2
+keyring==25.5.0
 kiwisolver==1.4.5
 langchain==0.2.12
 langchain-community==0.2.11
@@ -89,6 +100,7 @@ langchain-unstructured==0.1.1
 langdetect==1.0.9
 langsmith==0.1.98
 layoutparser==0.3.4
+llvmlite==0.43.0
 lxml==5.2.2
 Markdown==3.6
 markdown-it-py==3.0.0
@@ -97,6 +109,7 @@ marshmallow==3.21.3
 matplotlib==3.9.2
 matplotlib-inline==0.1.7
 mdurl==0.1.2
+more-itertools==10.5.0
 motor==3.4.0
 mpmath==1.3.0
 multidict==6.0.5
@@ -105,7 +118,10 @@ mypy-boto3-s3==1.34.138
 mypy-extensions==1.0.0
 nest-asyncio==1.6.0
 networkx==3.3
+nh3==0.2.20
 nltk==3.8.1
+nodeenv==1.9.1
+numba==0.60.0
 numpy==1.26.4
 olefile==0.47
 ollama==0.3.1
@@ -121,6 +137,7 @@ packaging==24.0
 pandas==2.2.2
 parso==0.8.4
 passlib==1.7.4
+pathspec==0.12.1
 pdf2image==1.17.0
 pdfminer.six==20231228
 pdfplumber==0.11.3
@@ -128,9 +145,11 @@ pexpect==4.9.0
 pikepdf==9.1.1
 pillow==10.4.0
 pillow_heif==0.18.0
+pkginfo==1.12.0
 platformdirs==4.2.2
 pluggy==1.5.0
 portalocker==2.10.1
+pre_commit==4.0.1
 prompt_toolkit==3.0.47
 proto-plus==1.24.0
 protobuf==5.27.3
@@ -152,8 +171,10 @@ pypandoc==1.13
 pyparsing==3.1.2
 pypdf==4.3.1
 pypdfium2==4.30.0
+pyproject_hooks==1.2.0
 pytesseract==0.3.10
 pytest==8.2.0
+pytest-asyncio==0.24.0
 python-dateutil==2.9.0.post0
 python-docx==1.1.2
 python-dotenv==1.0.1
@@ -168,9 +189,11 @@ PyYAML==6.0.1
 pyzmq==26.2.0
 rank-bm25==0.2.2
 rapidfuzz==3.9.5
+readme_renderer==44.0
 regex==2024.7.24
 requests==2.32.3
 requests-toolbelt==1.0.0
+rfc3986==2.0.0
 rich==13.7.1
 rsa==4.9
 s3transfer==0.10.1
@@ -189,12 +212,14 @@ tenacity==8.5.0
 tiktoken==0.7.0
 timm==1.0.8
 tokenizers==0.19.1
+tomli==2.2.1
 torch==2.2.2
 torchvision==0.17.2
 tornado==6.4.1
 tqdm==4.66.4
 traitlets==5.14.3
 transformers==4.44.0
+twine==6.0.1
 typer==0.12.3
 types-awscrt==0.21.2
 types-s3transfer==0.10.1
@@ -209,6 +234,7 @@ unstructured.pytesseract==0.3.12
 urllib3==2.2.1
 uvicorn==0.29.0
 uvloop==0.19.0
+virtualenv==20.28.0
 watchfiles==0.21.0
 wcwidth==0.2.13
 websockets==12.0