diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 0000000..5ad1394 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,31 @@ +name: "Documentation" + +on: [push, pull_request] + +jobs: + docs: + name: "Documentation" + runs-on: ubuntu-latest + steps: + - name: "Checkout Instaloader Repository" + uses: actions/checkout@v2 + with: + fetch-depth: 0 # needed for building docs + - name: "Setup Python" + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: "Install Dependencies" + run: | + python -m pip install pipenv==2022.1.8 + pipenv sync --dev + - name: "Build Documentation" + run: pipenv run make -C docs html SPHINXOPTS="-W -n" + - name: "Deploy Documentation" + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + uses: JamesIves/github-pages-deploy-action@v4.2.5 + with: + branch: master + repository-name: instaloader/instaloader.github.io + folder: docs/_build/html + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e1f72c6..15852da 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -8,24 +8,19 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] steps: - name: Checkout Instaloader Repository uses: actions/checkout@v2 - with: - fetch-depth: 0 # needed for building docs - name: Setup Python uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install Dependencies run: | - python -m pip install pipenv==2021.5.29 + python -m pip install pipenv==2022.1.8 pipenv sync --dev - name: PyLint run: pipenv run pylint instaloader - name: MyPy run: pipenv run mypy -m instaloader - - name: Build Documentation - if: matrix.python-version == '3.8' - run: pipenv run make -C docs html SPHINXOPTS="-W -n" diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml new file mode 100644 index 0000000..02c75a6 --- /dev/null +++ b/.github/workflows/packages.yml @@ -0,0 +1,40 @@ +name: "Publish Packages" + +on: + push: + tags: + - v* + +jobs: + publish: + name: "Publish Package" + runs-on: ubuntu-latest + steps: + - name: "Checkout Repository" + uses: actions/checkout@v2 + - name: "Setup Python" + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: "Get the tagged version" + id: get_version + env: + GITHUB_REF: ${{ github.ref }} + run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\/v/} + shell: bash + - name: "Install Dependencies" + run: | + python -m pip install pipenv==2022.1.8 + pipenv sync --dev + - name: "Generate Python Package" + run: python setup.py sdist + - name: "Publish Distribution to PyPI" + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_PASSWORD }} + - name: "Publish Distribution to AUR" + run: deploy/arch/deploy.sh $VERSION_TAG + env: + VERSION_TAG: v${{ steps.get_version.outputs.VERSION }} + AUR_KEY: ${{ secrets.AUR_KEY }} + AUR_IV: ${{ secrets.AUR_IV }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6b6e912..0000000 --- a/.travis.yml +++ /dev/null @@ -1,35 +0,0 @@ -dist: xenial -language: python -python: - - "3.6" - - "3.7" -install: - - pipenv sync --dev -script: - - pylint instaloader - - mypy -m instaloader - - make -C docs html SPHINXOPTS="-W -n" -addons: - ssh_known_hosts: aur.archlinux.org -deploy: - - provider: pypi - user: aandergr - on: - tags: true - python: 3.6 - password: - secure: KBsATtXNnEzqr0X2ALdyx2zaYj2EUfK8YjP4jEOilBELpJ4evHYXVgRkn9CKXGsD7LKbWmR3gz2rr8NZ6U42pHw0fpQHizz91MpJSKbNY7RZZh4UVWMDh81izeglchWAF+m3Wdpxpo/GBDbrZIbRQFhPgB+kjSzJo3uiCS+yVXtGG+6csJHzd0WkJjsVHz+XSdxyOiLy/qFF3nmY6ZrVrgw6WbX2YLbx9je6DxK3YRrnBqgMgH7Ui2xWWbgJCk3KKpQTMiMKnTa9mdeMJAd26YMCz1gG/X7ck6VNly/6bs5+Zq7/bU0dSolhPBhNthyccYdS0IeGPdPF3YqatblJ4fLGTfHn8oJs3/T11yIW9pvOT3UftEynPOUX+en2Qnb/3WfgwUfVts8i8DAYgEFLGEzbF5XMGy4gwAD/bZ0JW0pmgAG2PZsdOokAt+ydlNj/uxq1GFW2doZbul6k4t03p1mik6itYscvuhKoEm8Yw6wr1P9aDvMR6FFpSu4gDt2XrKmRmWkSXxpNeCfZujTjyAnISxLVQGSY7BG5K0Z2HbDtqqNz9nj4WgNGzZmngiRofCevWR+vPIDUDW7NCX03csfcFS9RRllBO/BrL7NVUV+Qp9dfTQYp+8SqYLWgHtDoXdgyMREjJaAmbx2u/TYTY8bADZ5F3vGUpv2j5hP3NTU= - - provider: pages - skip-cleanup: true - github-token: $GITHUB_TOKEN - local-dir: docs/_build/html - repo: instaloader/instaloader.github.io - target-branch: master - on: - branch: master - python: 3.6 - - provider: script - script: deploy/arch/deploy.sh $TRAVIS_TAG - on: - tags: true - python: 3.6 diff --git a/deploy/arch/deploy.sh b/deploy/arch/deploy.sh index 6de2982..e92db04 100755 --- a/deploy/arch/deploy.sh +++ b/deploy/arch/deploy.sh @@ -3,15 +3,16 @@ VERSION=${1:1} # do not deploy pre-releases -echo $VERSION | grep -qe "[abc]" && exit 0 +#echo $VERSION | grep -qe "[abc]" && exit 0 cd $(dirname $0) # decrypt and add ssh key -openssl aes-256-cbc -K $encrypted_354637631c28_key -iv $encrypted_354637631c28_iv -in id_rsa_AUR.enc -out /tmp/AUR_openssh -d +openssl aes-256-cbc -K $AUR_KEY -iv $AUR_IV -in id_rsa_AUR.enc -out /tmp/AUR_openssh -d eval "$(ssh-agent -s)" chmod 600 /tmp/AUR_openssh ssh-add /tmp/AUR_openssh +ssh-keyscan -t rsa aur.archlinux.org >> ~/.ssh/known_hosts # clone and modify AUR repo git clone --depth 1 ssh://aur@aur.archlinux.org/instaloader.git diff --git a/deploy/arch/id_rsa_AUR.enc b/deploy/arch/id_rsa_AUR.enc index cad2372..3d6e7d5 100644 Binary files a/deploy/arch/id_rsa_AUR.enc and b/deploy/arch/id_rsa_AUR.enc differ diff --git a/deploy/windows/create_exe.py b/deploy/windows/create_exe.py index 7e25bc3..d9a1daa 100644 --- a/deploy/windows/create_exe.py +++ b/deploy/windows/create_exe.py @@ -44,7 +44,7 @@ with open('__main__.py', 'w+') as f: f.writelines(lines) # install dependencies and invoke PyInstaller -commands = ["pip install pipenv==2021.5.29", +commands = ["pip install pipenv==2022.1.8", f"pipenv --python {sys.version_info.major}.{sys.version_info.minor} sync --dev", "pipenv run pyinstaller --log-level=DEBUG instaloader.spec"] diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index b23d96c..9371c3f 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -335,15 +335,25 @@ class Instaloader: filename_suffix: Optional[str] = None, _attempt: int = 1) -> bool: """Downloads and saves picture with given url under given directory with given timestamp. Returns true, if file was actually downloaded, i.e. updated.""" - urlmatch = re.search('\\.[a-z0-9]*\\?', url) - file_extension = url[-3:] if urlmatch is None else urlmatch.group(0)[1:-1] if filename_suffix is not None: filename += '_' + filename_suffix - filename += '.' + file_extension - if os.path.isfile(filename): + urlmatch = re.search('\\.[a-z0-9]*\\?', url) + file_extension = url[-3:] if urlmatch is None else urlmatch.group(0)[1:-1] + nominal_filename = filename + '.' + file_extension + if os.path.isfile(nominal_filename): + self.context.log(nominal_filename + ' exists', end=' ', flush=True) + return False + resp = self.context.get_raw(url) + if 'Content-Type' in resp.headers and resp.headers['Content-Type']: + header_extension = '.' + resp.headers['Content-Type'].split(';')[0].split('/')[-1] + header_extension = header_extension.lower().replace('jpeg', 'jpg') + filename += header_extension + else: + filename = nominal_filename + if filename != nominal_filename and os.path.isfile(filename): self.context.log(filename + ' exists', end=' ', flush=True) return False - self.context.get_and_write_raw(url, filename) + self.context.write_raw(resp, filename) os.utime(filename, (datetime.now().timestamp(), mtime.timestamp())) return True diff --git a/instaloader/structures.py b/instaloader/structures.py index c4c51c3..11f1da7 100644 --- a/instaloader/structures.py +++ b/instaloader/structures.py @@ -8,6 +8,7 @@ from datetime import datetime from itertools import islice from pathlib import Path from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple, Union +from unicodedata import normalize from . import __version__ from .exceptions import * @@ -365,9 +366,9 @@ class Post: def caption(self) -> Optional[str]: """Caption.""" if "edge_media_to_caption" in self._node and self._node["edge_media_to_caption"]["edges"]: - return self._node["edge_media_to_caption"]["edges"][0]["node"]["text"] + return normalize("NFC", self._node["edge_media_to_caption"]["edges"][0]["node"]["text"]) elif "caption" in self._node: - return self._node["caption"] + return normalize("NFC", self._node["caption"]) return None @property @@ -389,7 +390,7 @@ class Post: # support Unicode and a word/beginning of string delimiter at the beginning to ensure # that no email addresses join the list of mentions. # http://blog.jstassen.com/2016/03/code-regex-for-instagram-username-and-hashtags/ - mention_regex = re.compile(r"(?:^|\W|_)(?:@)(\w(?:(?:\w|(?:\.(?!\.))){0,28}(?:\w))?)") + mention_regex = re.compile(r"(?:^|\W|_)(?:@)(\w(?:(?:\w|(?:\.(?!\.))){0,28}(?:\w))?)", re.ASCII) return re.findall(mention_regex, self.caption.lower()) @property