diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
new file mode 100644
index 0000000000000000000000000000000000000000..49a7bb6232aa26df0fe2119c1a47d2e38a759020
--- /dev/null
+++ b/.azure-pipelines/ci.yml
@@ -0,0 +1,136 @@
+variables:
+ manylinux: false
+ coverage: false
+
+jobs:
+- job: Prebuild
+ displayName: Pre-build checks
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ steps:
+ - template: ./prebuild-checks.yml
+
+
+- job: Docs_PR
+ displayName: Docs PR
+ dependsOn: Prebuild
+ condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true'))
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ steps:
+ - template: ./docs-steps.yml
+ parameters:
+ upload: true
+
+
+- job: macOS_CI_Tests
+ displayName: macOS CI Tests
+ dependsOn: Prebuild
+ condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+ variables:
+ testRunTitle: '$(build.sourceBranchName)-macos'
+ testRunPlatform: macos
+
+ pool:
+ vmImage: xcode9-macos10.13
+
+ steps:
+ - template: ./macos-steps.yml
+
+
+- job: Ubuntu_CI_Tests
+ displayName: Ubuntu CI Tests
+ dependsOn: Prebuild
+ condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ variables:
+ testRunTitle: '$(build.sourceBranchName)-linux'
+ testRunPlatform: linux
+ openssl_version: 1.1.0g
+
+ steps:
+ - template: ./posix-steps.yml
+
+
+- job: ManyLinux1_CI_Tests
+ displayName: ManyLinux1 CI Tests
+ dependsOn: Prebuild
+ condition: |
+ and(
+ and(
+ succeeded(),
+ eq(variables['manylinux'], 'true')
+ ),
+ eq(dependencies.Prebuild.outputs['tests.run'], 'true')
+ )
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ variables:
+ testRunTitle: '$(build.sourceBranchName)-manylinux1'
+ testRunPlatform: manylinux1
+ imageName: 'dockcross/manylinux-x64'
+
+ steps:
+ - template: ./docker-steps.yml
+
+
+- job: Ubuntu_Coverage_CI_Tests
+ displayName: Ubuntu CI Tests (coverage)
+ dependsOn: Prebuild
+ condition: |
+ and(
+ and(
+ succeeded(),
+ eq(variables['coverage'], 'true')
+ ),
+ eq(dependencies.Prebuild.outputs['tests.run'], 'true')
+ )
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ variables:
+ testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
+ testRunPlatform: linux-coverage
+ openssl_version: 1.1.0g
+
+ steps:
+ - template: ./posix-steps.yml
+ parameters:
+ coverage: true
+
+
+- job: Windows_CI_Tests
+ displayName: Windows CI Tests
+ dependsOn: Prebuild
+ condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+ pool:
+ vmImage: vs2017-win2016
+
+ strategy:
+ matrix:
+ win32:
+ arch: win32
+ buildOpt:
+ testRunTitle: '$(Build.SourceBranchName)-win32'
+ testRunPlatform: win32
+ win64:
+ arch: amd64
+ buildOpt: '-p x64'
+ testRunTitle: '$(Build.SourceBranchName)-win64'
+ testRunPlatform: win64
+ maxParallel: 2
+
+ steps:
+ - template: ./windows-steps.yml
diff --git a/.azure-pipelines/docker-steps.yml b/.azure-pipelines/docker-steps.yml
new file mode 100644
index 0000000000000000000000000000000000000000..ba4dfd72dd8bb8bbadf212fe17799578802c9cb6
--- /dev/null
+++ b/.azure-pipelines/docker-steps.yml
@@ -0,0 +1,76 @@
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+- ${{ if ne(parameters.targetBranch, '') }}:
+ - script: |
+ git fetch -q origin ${{ parameters.targetbranch }}
+ if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)'
+ then
+ echo "Only docs were updated, stopping build process."
+ echo "##vso[task.setvariable variable=DocOnly]true"
+ exit
+ fi
+ displayName: Detect doc-only changes
+
+- task: docker@0
+ displayName: 'Configure CPython (debug)'
+ inputs:
+ action: 'Run an image'
+ imageName: $(imageName)
+ volumes: |
+ $(build.sourcesDirectory):/src
+ $(build.binariesDirectory):/build
+ workDir: '/src'
+ containerCommand: './configure --with-pydebug'
+ detached: false
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- task: docker@0
+ displayName: 'Build CPython'
+ inputs:
+ action: 'Run an image'
+ imageName: $(imageName)
+ volumes: |
+ $(build.sourcesDirectory):/src
+ $(build.binariesDirectory):/build
+ workDir: '/src'
+ containerCommand: 'make -s -j4'
+ detached: false
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- task: docker@0
+ displayName: 'Display build info'
+ inputs:
+ action: 'Run an image'
+ imageName: $(imageName)
+ volumes: |
+ $(build.sourcesDirectory):/src
+ $(build.binariesDirectory):/build
+ workDir: '/src'
+ containerCommand: 'make pythoninfo'
+ detached: false
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- task: docker@0
+ displayName: 'Tests'
+ inputs:
+ action: 'Run an image'
+ imageName: $(imageName)
+ volumes: |
+ $(build.sourcesDirectory):/src
+ $(build.binariesDirectory):/build
+ workDir: '/src'
+ containerCommand: 'make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=/build/test-results.xml"'
+ detached: false
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- task: PublishTestResults@2
+ displayName: 'Publish Test Results'
+ inputs:
+ testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
+ mergeTestResults: true
+ testRunTitle: $(testRunTitle)
+ platform: $(testRunPlatform)
+ condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true'))
diff --git a/.azure-pipelines/docs-steps.yml b/.azure-pipelines/docs-steps.yml
new file mode 100644
index 0000000000000000000000000000000000000000..492e4e34bb2dab9c6f3631858a238cec93ca8798
--- /dev/null
+++ b/.azure-pipelines/docs-steps.yml
@@ -0,0 +1,46 @@
+parameters:
+ latex: false
+ upload: false
+
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+- task: UsePythonVersion@0
+ displayName: 'Use Python 3.6 or later'
+ inputs:
+ versionSpec: '>=3.6'
+
+- script: python -m pip install sphinx==1.8.2 blurb python-docs-theme
+ displayName: 'Install build dependencies'
+
+- ${{ if ne(parameters.latex, 'true') }}:
+ - script: make check suspicious html PYTHON=python
+ workingDirectory: '$(build.sourcesDirectory)/Doc'
+ displayName: 'Build documentation'
+
+- ${{ if eq(parameters.latex, 'true') }}:
+ - script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full
+ displayName: 'Install LaTeX'
+
+ - script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb'
+ workingDirectory: '$(build.sourcesDirectory)/Doc'
+ displayName: 'Build documentation'
+
+- ${{ if eq(parameters.upload, 'true') }}:
+ - task: PublishBuildArtifacts@1
+ displayName: 'Publish docs'
+
+ inputs:
+ PathToPublish: '$(build.sourcesDirectory)/Doc/build'
+ ArtifactName: docs
+ publishLocation: Container
+
+ - ${{ if eq(parameters.latex, 'true') }}:
+ - task: PublishBuildArtifacts@1
+ displayName: 'Publish dist'
+ inputs:
+ PathToPublish: '$(build.sourcesDirectory)/Doc/dist'
+ ArtifactName: docs_dist
+ publishLocation: Container
diff --git a/.azure-pipelines/macos-steps.yml b/.azure-pipelines/macos-steps.yml
new file mode 100644
index 0000000000000000000000000000000000000000..647081689454acc3e64822b28cb04773303c0b55
--- /dev/null
+++ b/.azure-pipelines/macos-steps.yml
@@ -0,0 +1,25 @@
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-azdev
+ displayName: 'Configure CPython (debug)'
+
+- script: make -s -j4
+ displayName: 'Build CPython'
+
+- script: make pythoninfo
+ displayName: 'Display build info'
+
+- script: make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
+ displayName: 'Tests'
+
+- task: PublishTestResults@2
+ displayName: 'Publish Test Results'
+ inputs:
+ testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
+ mergeTestResults: true
+ testRunTitle: $(testRunTitle)
+ platform: $(testRunPlatform)
+ condition: succeededOrFailed()
diff --git a/.azure-pipelines/posix-deps.sh b/.azure-pipelines/posix-deps.sh
new file mode 100755
index 0000000000000000000000000000000000000000..a572107566016ec5bca7ddd5540de9c1e7e2a206
--- /dev/null
+++ b/.azure-pipelines/posix-deps.sh
@@ -0,0 +1,26 @@
+sudo apt-get update
+
+sudo apt-get -yq install \
+ build-essential \
+ zlib1g-dev \
+ libbz2-dev \
+ liblzma-dev \
+ libncurses5-dev \
+ libreadline6-dev \
+ libsqlite3-dev \
+ libssl-dev \
+ libgdbm-dev \
+ tk-dev \
+ lzma \
+ lzma-dev \
+ liblzma-dev \
+ libffi-dev \
+ uuid-dev \
+ xvfb
+
+if [ ! -z "$1" ]
+then
+ echo ##vso[task.prependpath]$PWD/multissl/openssl/$1
+ echo ##vso[task.setvariable variable=OPENSSL_DIR]$PWD/multissl/openssl/$1
+ python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $1 --system Linux
+fi
diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml
new file mode 100644
index 0000000000000000000000000000000000000000..9fec9be8014ffd6cda5fa8e15e333c7ccad8a472
--- /dev/null
+++ b/.azure-pipelines/posix-steps.yml
@@ -0,0 +1,63 @@
+parameters:
+ coverage: false
+
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+- script: ./.azure-pipelines/posix-deps.sh $(openssl_version)
+ displayName: 'Install dependencies'
+
+- script: ./configure --with-pydebug
+ displayName: 'Configure CPython (debug)'
+
+- script: make -s -j4
+ displayName: 'Build CPython'
+
+- ${{ if eq(parameters.coverage, 'true') }}:
+ - script: ./python -m venv venv && ./venv/bin/python -m pip install -U coverage
+ displayName: 'Set up virtual environment'
+
+ - script: ./venv/bin/python -m test.pythoninfo
+ displayName: 'Display build info'
+
+ - script: |
+ xvfb-run ./venv/bin/python -m coverage run --pylib -m test \
+ --fail-env-changed \
+ -uall,-cpu \
+ --junit-xml=$(build.binariesDirectory)/test-results.xml" \
+ -x test_multiprocessing_fork \
+ -x test_multiprocessing_forkserver \
+ -x test_multiprocessing_spawn \
+ -x test_concurrent_futures
+ displayName: 'Tests with coverage'
+
+ - script: ./venv/bin/python -m coverage xml
+ displayName: 'Generate coverage.xml'
+
+ - script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash)
+ displayName: 'Publish code coverage results'
+
+
+- ${{ if ne(parameters.coverage, 'true') }}:
+ - script: make pythoninfo
+ displayName: 'Display build info'
+
+ - script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
+ displayName: 'Tests'
+
+
+- script: ./python Tools/scripts/patchcheck.py --travis true
+ displayName: 'Run patchcheck.py'
+ condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
+
+
+- task: PublishTestResults@2
+ displayName: 'Publish Test Results'
+ inputs:
+ testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
+ mergeTestResults: true
+ testRunTitle: $(testRunTitle)
+ platform: $(testRunPlatform)
+ condition: succeededOrFailed()
diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml
new file mode 100644
index 0000000000000000000000000000000000000000..2d7fba9cf3283ed8988342bd97dcc0dbc5eb8d57
--- /dev/null
+++ b/.azure-pipelines/pr.yml
@@ -0,0 +1,86 @@
+jobs:
+- job: Prebuild
+ displayName: Pre-build checks
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ steps:
+ - template: ./prebuild-checks.yml
+
+
+- job: Docs_PR
+ displayName: Docs PR
+ dependsOn: Prebuild
+ condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true'))
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ steps:
+ - template: ./docs-steps.yml
+
+
+- job: macOS_PR_Tests
+ displayName: macOS PR Tests
+ dependsOn: Prebuild
+ condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+ variables:
+ testRunTitle: '$(system.pullRequest.TargetBranch)-macos'
+ testRunPlatform: macos
+
+ pool:
+ vmImage: xcode9-macos10.13
+
+ steps:
+ - template: ./macos-steps.yml
+ parameters:
+ targetBranch: $(System.PullRequest.TargetBranch)
+
+
+- job: Ubuntu_PR_Tests
+ displayName: Ubuntu PR Tests
+ dependsOn: Prebuild
+ condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ variables:
+ testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
+ testRunPlatform: linux
+ openssl_version: 1.1.0g
+
+ steps:
+ - template: ./posix-steps.yml
+ parameters:
+ targetBranch: $(System.PullRequest.TargetBranch)
+
+
+- job: Windows_PR_Tests
+ displayName: Windows PR Tests
+ dependsOn: Prebuild
+ condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+ pool:
+ vmImage: vs2017-win2016
+
+ strategy:
+ matrix:
+ win32:
+ arch: win32
+ buildOpt:
+ testRunTitle: '$(System.PullRequest.TargetBranch)-win32'
+ testRunPlatform: win32
+ win64:
+ arch: amd64
+ buildOpt: '-p x64'
+ testRunTitle: '$(System.PullRequest.TargetBranch)-win64'
+ testRunPlatform: win64
+ maxParallel: 2
+
+ steps:
+ - template: ./windows-steps.yml
+ parameters:
+ targetBranch: $(System.PullRequest.TargetBranch)
diff --git a/.azure-pipelines/prebuild-checks.yml b/.azure-pipelines/prebuild-checks.yml
new file mode 100644
index 0000000000000000000000000000000000000000..30ff642d1267a1eda853bac55cbe5a1493af64cb
--- /dev/null
+++ b/.azure-pipelines/prebuild-checks.yml
@@ -0,0 +1,36 @@
+steps:
+- checkout: self
+ fetchDepth: 5
+
+- script: echo "##vso[task.setvariable variable=diffTarget]HEAD~1"
+ displayName: Set default diff target
+
+- script: |
+ git fetch -q origin $(System.PullRequest.TargetBranch)
+ echo "##vso[task.setvariable variable=diffTarget]HEAD \$(git merge-base HEAD FETCH_HEAD)"
+ displayName: Fetch comparison tree
+ condition: and(succeeded(), variables['System.PullRequest.TargetBranch'])
+
+- script: |
+ if ! git diff --name-only $(diffTarget) | grep -qE '(\.rst$|^Doc|^Misc)'
+ then
+ echo "No docs were updated: docs.run=false"
+ echo "##vso[task.setvariable variable=run;isOutput=true]false"
+ else
+ echo "Docs were updated: docs.run=true"
+ echo "##vso[task.setvariable variable=run;isOutput=true]true"
+ fi
+ displayName: Detect documentation changes
+ name: docs
+
+- script: |
+ if ! git diff --name-only $(diffTarget) | grep -qvE '(\.rst$|^Doc|^Misc)'
+ then
+ echo "Only docs were updated: tests.run=false"
+ echo "##vso[task.setvariable variable=run;isOutput=true]false"
+ else
+ echo "Code was updated: tests.run=true"
+ echo "##vso[task.setvariable variable=run;isOutput=true]true"
+ fi
+ displayName: Detect source changes
+ name: tests
diff --git a/.azure-pipelines/windows-appx-test.yml b/.azure-pipelines/windows-appx-test.yml
new file mode 100644
index 0000000000000000000000000000000000000000..5f3fe6c9457890bd9eac84ac8bb0b9ae069a45a8
--- /dev/null
+++ b/.azure-pipelines/windows-appx-test.yml
@@ -0,0 +1,67 @@
+jobs:
+- job: Prebuild
+ displayName: Pre-build checks
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ steps:
+ - template: ./prebuild-checks.yml
+
+
+- job: Windows_Appx_Tests
+ displayName: Windows Appx Tests
+ dependsOn: Prebuild
+ condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+ pool:
+ vmImage: vs2017-win2016
+
+ strategy:
+ matrix:
+ win64:
+ arch: amd64
+ buildOpt: '-p x64'
+ testRunTitle: '$(Build.SourceBranchName)-win64-appx'
+ testRunPlatform: win64
+ maxParallel: 2
+
+ steps:
+ - checkout: self
+ clean: true
+ fetchDepth: 5
+
+ - powershell: |
+ # Relocate build outputs outside of source directory to make cleaning faster
+ Write-Host '##vso[task.setvariable variable=Py_IntDir]$(Build.BinariesDirectory)\obj'
+ # UNDONE: Do not build to a different directory because of broken tests
+ Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.SourcesDirectory)\PCbuild'
+ Write-Host '##vso[task.setvariable variable=EXTERNAL_DIR]$(Build.BinariesDirectory)\externals'
+ displayName: Update build locations
+
+ - script: PCbuild\build.bat -e $(buildOpt)
+ displayName: 'Build CPython'
+ env:
+ IncludeUwp: true
+
+ - script: python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Py_IntDir)\layout-tmp-$(arch)" --copy "$(Py_IntDir)\layout-$(arch)" --precompile --preset-appx --include-tests
+ displayName: 'Create APPX layout'
+
+ - script: .\python.exe -m test.pythoninfo
+ workingDirectory: $(Py_IntDir)\layout-$(arch)
+ displayName: 'Display build info'
+
+ - script: .\python.exe -m test -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml" --tempdir "$(Py_IntDir)\tmp-$(arch)"
+ workingDirectory: $(Py_IntDir)\layout-$(arch)
+ displayName: 'Tests'
+ env:
+ PREFIX: $(Py_IntDir)\layout-$(arch)
+
+ - task: PublishTestResults@2
+ displayName: 'Publish Test Results'
+ inputs:
+ testResultsFiles: '$(Build.BinariesDirectory)\test-results.xml'
+ mergeTestResults: true
+ testRunTitle: $(testRunTitle)
+ platform: $(testRunPlatform)
+ condition: succeededOrFailed()
diff --git a/.azure-pipelines/windows-steps.yml b/.azure-pipelines/windows-steps.yml
new file mode 100644
index 0000000000000000000000000000000000000000..cba00158ad131dd020b288ac4dbb81129fee7cf7
--- /dev/null
+++ b/.azure-pipelines/windows-steps.yml
@@ -0,0 +1,34 @@
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+- powershell: |
+ # Relocate build outputs outside of source directory to make cleaning faster
+ Write-Host '##vso[task.setvariable variable=Py_IntDir]$(Build.BinariesDirectory)\obj'
+ # UNDONE: Do not build to a different directory because of broken tests
+ Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.SourcesDirectory)\PCbuild'
+ Write-Host '##vso[task.setvariable variable=EXTERNAL_DIR]$(Build.BinariesDirectory)\externals'
+ displayName: Update build locations
+
+- script: PCbuild\build.bat -e $(buildOpt)
+ displayName: 'Build CPython'
+ env:
+ IncludeUwp: true
+
+- script: python.bat -m test.pythoninfo
+ displayName: 'Display build info'
+
+- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml" --tempdir="$(Build.BinariesDirectory)\test"
+ displayName: 'Tests'
+ env:
+ PREFIX: $(Py_OutDir)\$(arch)
+
+- task: PublishTestResults@2
+ displayName: 'Publish Test Results'
+ inputs:
+ testResultsFiles: '$(Build.BinariesDirectory)\test-results.xml'
+ mergeTestResults: true
+ testRunTitle: $(testRunTitle)
+ platform: $(testRunPlatform)
+ condition: succeededOrFailed()
diff --git a/.bzrignore b/.bzrignore
deleted file mode 100644
index 58471109208922c9ee8c4b06135725f03ed16814..0000000000000000000000000000000000000000
--- a/.bzrignore
+++ /dev/null
@@ -1,42 +0,0 @@
-.purify
-autom4te.cache
-config.log
-config.cache
-config.status
-config.status.lineno
-db_home
-Makefile
-buildno
-python
-build
-Makefile.pre
-platform
-pybuilddir.txt
-pyconfig.h
-libpython*.a
-libpython*.so*
-python.exe
-python-gdb.py
-reflog.txt
-tags
-TAGS
-.gdb_history
-Doc/tools/sphinx
-Doc/tools/jinja
-Doc/tools/jinja2
-Doc/tools/pygments
-Doc/tools/docutils
-Misc/python.pc
-Modules/Setup
-Modules/Setup.config
-Modules/Setup.local
-Modules/config.c
-Modules/ld_so_aix
-Parser/pgen
-Lib/test/data/*
-Lib/lib2to3/Grammar*.pickle
-Lib/lib2to3/PatternGrammar*.pickle
-__pycache__
-.coverage
-coverage/*
-htmlcov/*
diff --git a/.gitattributes b/.gitattributes
index 5eead66489830d12d76a67e100c63823d595f91b..16237bb2b3ac19f7d8ea450fe0de72a8b3467a44 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -19,6 +19,7 @@
# Specific binary files
Lib/test/sndhdrdata/sndhdr.* binary
+PC/classicAppCompat.* binary
# Text files that should not be subject to eol conversion
Lib/test/cjkencodings/* -text
@@ -26,9 +27,7 @@ Lib/test/decimaltestdata/*.decTest -text
Lib/test/test_email/data/*.txt -text
Lib/test/xmltestdata/* -text
Lib/test/coding20731.py -text
-
-# Special files in third party code
-Modules/zlib/zlib.map -text
+Lib/test/test_importlib/data01/* -text
# CRLF files
*.bat text eol=crlf
@@ -39,3 +38,20 @@ Modules/zlib/zlib.map -text
*.proj text eol=crlf
PCbuild/readme.txt text eol=crlf
PC/readme.txt text eol=crlf
+
+# Generated files
+# https://github.com/github/linguist#generated-code
+Include/graminit.h linguist-generated=true
+Python/graminit.h linguist-generated=true
+Modules/clinic/*.h linguist-generated=true
+Objects/clinic/*.h linguist-generated=true
+PC/clinic/*.h linguist-generated=true
+Python/clinic/*.h linguist-generated=true
+Python/importlib.h linguist-generated=true
+Python/importlib_external.h linguist-generated=true
+Include/Python-ast.h linguist-generated=true
+Python/Python-ast.c linguist-generated=true
+Include/opcode.h linguist-generated=true
+Python/opcode_targets.h linguist-generated=true
+Objects/typeslots.inc linguist-generated=true
+Modules/unicodedata_db.h linguist-generated=true
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
new file mode 100644
index 0000000000000000000000000000000000000000..17d7ef4d7a243ef4ea8ccc31889684359f6c52e7
--- /dev/null
+++ b/.github/CODEOWNERS
@@ -0,0 +1,64 @@
+# See https://help.github.com/articles/about-codeowners/
+# for more info about CODEOWNERS file
+
+# It uses the same pattern rule for gitignore file
+# https://git-scm.com/docs/gitignore#_pattern_format
+
+# asyncio
+**/*asyncio* @1st1 @asvetlov
+
+# Core
+**/*context* @1st1
+**/*genobject* @1st1
+**/*hamt* @1st1
+
+# Hashing
+**/*hashlib* @python/crypto-team
+**/*pyhash* @python/crypto-team
+
+# Import (including importlib).
+# Ignoring importlib.h so as to not get flagged on
+# all pull requests that change the the emitted
+# bytecode.
+**/*import*.c @python/import-team
+**/*import*.py @python/import-team
+
+
+# SSL
+**/*ssl* @python/crypto-team
+
+# CSPRNG
+Python/bootstrap_hash.c @python/crypto-team
+
+# Email and related
+**/*mail* @python/email-team
+**/*smtp* @python/email-team
+**/*mime* @python/email-team
+**/*imap* @python/email-team
+**/*poplib* @python/email-team
+
+# subprocess
+**/*subprocess* @gpshead
+
+# Windows
+/PC/ @python/windows-team
+/PCbuild/ @python/windows-team
+
+# Windows installer packages
+/Tools/msi/ @python/windows-team
+/Tools/nuget/ @python/windows-team
+
+**/*itertools* @rhettinger
+**/*collections* @rhettinger
+**/*random* @rhettinger
+**/*queue* @rhettinger
+**/*bisect* @rhettinger
+**/*heapq* @rhettinger
+**/*functools* @ncoghlan @rhettinger
+**/*decimal* @rhettinger @skrah
+
+**/*dataclasses* @ericvsmith
+
+**/*idlelib* @terryjreedy
+
+**/*typing* @gvanrossum @ilevkivskyi
diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst
new file mode 100644
index 0000000000000000000000000000000000000000..7f912e870841902f6fcf3d4e1c2f225ab7e26b27
--- /dev/null
+++ b/.github/CONTRIBUTING.rst
@@ -0,0 +1,67 @@
+Contributing to Python
+======================
+
+Build Status
+------------
+
+- master
+
+ + `Stable buildbots `_
+
+- 3.7
+
+ + `Stable buildbots `_
+
+- 3.6
+
+ + `Stable buildbots `_
+
+- 2.7
+
+ + `Stable buildbots `_
+
+
+Thank You
+---------
+First off, thanks for contributing to the maintenance of the Python programming
+language and the CPython interpreter! Even if your contribution is not
+ultimately accepted, the fact you put time and effort into helping out is
+greatly appreciated.
+
+
+Contribution Guidelines
+-----------------------
+Please read the `devguide `_ for
+guidance on how to contribute to this project. The documentation covers
+everything from how to build the code to submitting a pull request. There are
+also suggestions on how you can most effectively help the project.
+
+Please be aware that our workflow does deviate slightly from the typical GitHub
+project. Details on how to properly submit a pull request are covered in
+`Lifecycle of a Pull Request `_.
+We utilize various bots and status checks to help with this, so do follow the
+comments they leave and their "Details" links, respectively. The key points of
+our workflow that are not covered by a bot or status check are:
+
+- All discussions that are not directly related to the code in the pull request
+ should happen on bugs.python.org
+- Upon your first non-trivial pull request (which includes documentation changes),
+ feel free to add yourself to ``Misc/ACKS``
+
+
+Setting Expectations
+--------------------
+Due to the fact that this project is entirely volunteer-run (i.e. no one is paid
+to work on Python full-time), we unfortunately can make no guarantees as to if
+or when a core developer will get around to reviewing your pull request.
+If no core developer has done a review or responded to changes made because of a
+"changes requested" review, please feel free to email python-dev to ask if
+someone could take a look at your pull request.
+
+
+Code of Conduct
+---------------
+All interactions for this project are covered by the
+`PSF Code of Conduct `_. Everyone is
+expected to be open, considerate, and respectful of others no matter their
+position within the project.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 4ce80d872df7b6f5d08912a268d6d952fa292e41..55e4168747e10d1e89ad5936bcbf656c28f7aea5 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,9 +1,12 @@
-## CPython Mirror
+!!! If this is a backport PR (PR made against branches other than `master`),
+please ensure that the PR title is in the following format:
-https://github.com/python/cpython is a cpython mirror repository. Pull requests
-are not accepted on this repo and will be automatically closed.
+```
+[X.Y]
(GH-NNNN)
+```
-### Submit patches at https://bugs.python.org
+Where: [X.Y] is the branch name, e.g. [3.7].
-For additional information about contributing to CPython, see the
-[developer's guide](https://docs.python.org/devguide/#contributing).
+GH-NNNN refers to the PR number from `master`.
+
+PLEASE: Remove this headline!!!
diff --git a/.github/appveyor.yml b/.github/appveyor.yml
index 09eb7db0f0d46d22b2bb3bd8e4aaf47b6734e396..6662732326a3b8e10c78689bdbdf7bdbff745a19 100644
--- a/.github/appveyor.yml
+++ b/.github/appveyor.yml
@@ -1,4 +1,4 @@
-version: 3.6build{build}
+version: 3.7build{build}
clone_depth: 5
branches:
only:
@@ -31,9 +31,8 @@ build_script:
- cmd: PCbuild\build.bat -e
- cmd: PCbuild\win32\python.exe -m test.pythoninfo
test_script:
- - cmd: PCbuild\rt.bat -q -uall -u-cpu -u-largefile -rwW --slowest --timeout=1200 -j0
+ - cmd: PCbuild\rt.bat -q -uall -u-cpu -u-largefile -rwW --slowest --timeout=1200 --fail-env-changed -j0
environment:
HOST_PYTHON: C:\Python36\python.exe
image:
- - Visual Studio 2015
- Visual Studio 2017
diff --git a/.github/codecov.yml b/.github/codecov.yml
index dc21321d0baaf0cd0192c9cf030f6f3ddcd2be50..9d97dfbc43f8d018eafcec38cb14be99fb8f5da8 100644
--- a/.github/codecov.yml
+++ b/.github/codecov.yml
@@ -13,18 +13,12 @@ ignore:
- "Grammar/*"
coverage:
precision: 2
- range:
- - 70.0
- - 100.0
+ range: 70...90
round: down
status:
changes: off
project: off
- patch:
- default:
- target: 100%
- only_pulls: true
- threshold: 0.05
+ patch: off
parsers:
gcov:
branch_detection:
diff --git a/.gitignore b/.gitignore
index 945ee76c609af2fca642c1ccea6fb82c88ea9463..58f8bf72f2b9bd5fa258e7110bffd973cf1d6a6d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1,16 @@
+# added for local development
+.buildaix/
+Modules/python.exp
+buildaix/
+installp/
+.gitignore
+
# Two-trick pony for OSX and other case insensitive file systems:
# Ignore ./python binary on Unix but still look into ./Python/ directory.
/python
!/Python/
*.cover
+*.iml
*.o
*.orig
*.pyc
@@ -35,7 +43,9 @@ Modules/Setup.local
Modules/config.c
Modules/ld_so_aix
Programs/_freeze_importlib
+Programs/_freeze_importlib.exe
Programs/_testembed
+Programs/_testembed.exe
PC/python_nt*.h
PC/pythonnt_rc*.h
PC/*/*.exe
@@ -63,6 +73,7 @@ PCbuild/obj/
PCbuild/win32/
.purify
Parser/pgen
+Parser/pgen.exe
__pycache__
autom4te.cache
build/
@@ -74,10 +85,12 @@ config.status.lineno
core
db_home
.hg/
+.idea/
ipch/
libpython*.a
libpython*.so*
libpython*.dylib
+libpython*.dll
platform
pybuilddir.txt
pyconfig.h
diff --git a/.hgignore b/.hgignore
deleted file mode 100644
index 92896b7bc21c152ca0c66d04b096f06508134cd4..0000000000000000000000000000000000000000
--- a/.hgignore
+++ /dev/null
@@ -1,105 +0,0 @@
-.gdb_history
-.purify
-.svn/
-^.idea/
-^.vscode/
-.DS_Store
-Makefile$
-Makefile.pre$
-TAGS$
-autom4te.cache$
-^build/
-^Doc/build/
-^Doc/venv/
-buildno$
-config.cache
-config.log
-config.status
-config.status.lineno
-db_home
-platform$
-pyconfig.h$
-python$
-python.bat$
-python.exe$
-python-config$
-python-config.py$
-reflog.txt$
-tags$
-Misc/python.pc
-Misc/python-config.sh$
-Modules/Setup$
-Modules/Setup.config
-Modules/Setup.local
-Modules/config.c
-Modules/ld_so_aix$
-Parser/pgen$
-^lcov-report/
-^core
-^python-gdb.py
-^python.exe-gdb.py
-^pybuilddir.txt
-
-syntax: glob
-libpython*.a
-libpython*.so*
-libpython*.dylib
-*.swp
-*.o
-*.pyc
-*.pyo
-*.pyd
-*.cover
-*~
-*.gc??
-*.profclang?
-*.profraw
-*.dyn
-Include/pydtrace_probes.h
-Lib/distutils/command/*.pdb
-Lib/lib2to3/*.pickle
-Lib/test/data/*
-Misc/*.wpu
-PC/python_nt*.h
-PC/pythonnt_rc*.h
-PC/*/*.exe
-PC/*/*.exp
-PC/*/*.lib
-PC/*/*.bsc
-PC/*/*.dll
-PC/*/*.pdb
-PC/*/*.user
-PC/*/*.ncb
-PC/*/*.suo
-PC/*/Win32-temp-*
-PC/*/x64-temp-*
-PC/*/amd64
-PCbuild/*.user
-PCbuild/*.suo
-PCbuild/*.*sdf
-PCbuild/*-pgi
-PCbuild/*-pgo
-PCbuild/.vs
-PCbuild/amd64
-PCbuild/obj
-PCbuild/win32
-Tools/unicode/build/
-Tools/unicode/MAPPINGS/
-BuildLog.htm
-__pycache__
-Programs/_freeze_importlib
-Programs/_testembed
-.coverage
-coverage/
-externals/
-htmlcov/
-*.gcda
-*.gcno
-*.gcov
-ipch/
-coverage.info
-Tools/msi/obj
-Tools/ssl/amd64
-Tools/ssl/win32
-.vs/
-.vscode/
diff --git a/.travis.yml b/.travis.yml
index 2ced4d0b97f6f84da251b2e5a6c2cfd2bc95cf29..f44cc741ef461918f4df6203aaf2fe01498e9047 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,11 +7,20 @@ group: beta
cache:
- pip
- ccache
+ - directories:
+ - $HOME/multissl
env:
global:
+ - OPENSSL=1.1.0i
+ - OPENSSL_DIR="$HOME/multissl/openssl/${OPENSSL}"
+ - PATH="${OPENSSL_DIR}/bin:$PATH"
# Use -O3 because we don't use debugger on Travis-CI
- - CFLAGS="-O3"
+ - CFLAGS="-I${OPENSSL_DIR}/include -O3"
+ - LDFLAGS="-L${OPENSSL_DIR}/lib"
+ # Set rpath with env var instead of -Wl,-rpath linker flag
+ # OpenSSL ignores LDFLAGS when linking bin/openssl
+ - LD_RUN_PATH="${OPENSSL_DIR}/lib"
branches:
only:
@@ -31,23 +40,32 @@ matrix:
# compiler here and the other to run the coverage build. Clang is preferred
# in this instance for its better error messages.
env: TESTING=cpython
+ addons:
+ apt:
+ packages:
+ - xvfb
- os: linux
language: python
+ # Build the docs against a stable version of Python so code bugs don't hold up doc-related PRs.
python: 3.6
env: TESTING=docs
before_script:
- cd Doc
# Sphinx is pinned so that new versions that introduce new warnings won't suddenly cause build failures.
# (Updating the version is fine as long as no warnings are raised by doing so.)
- - python -m pip install sphinx~=1.6.1 blurb
+ - python -m pip install sphinx==1.8.2 blurb
script:
- make check suspicious html SPHINXOPTS="-q -W -j4"
- os: linux
language: c
compiler: gcc
env: OPTIONAL=true
+ addons:
+ apt:
+ packages:
+ - xvfb
before_script:
- - ./configure PYTHON_FOR_REGEN=python3
+ - ./configure
- make -s -j4
# Need a venv that can parse covered code.
- ./python -m venv venv
@@ -55,7 +73,7 @@ matrix:
- ./venv/bin/python -m test.pythoninfo
script:
# Skip tests that re-run the entire test suite.
- - ./venv/bin/python -m coverage run --pylib -m test -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn
+ - xvfb-run ./venv/bin/python -m coverage run --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures
after_script: # Probably should be after_success once test suite updated to run under coverage.py.
# Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files.
- source ./venv/bin/activate
@@ -88,9 +106,21 @@ before_install:
fi
fi
+install:
+ - |
+ # Install OpenSSL as necessary
+ if [ "${TESTING}" != "docs" ]
+ then
+ # clang complains about unused-parameter a lot, redirect stderr
+ python3 Tools/ssl/multissltests.py --steps=library \
+ --base-directory ${HOME}/multissl \
+ --openssl ${OPENSSL} >/dev/null 2>&1
+ fi
+ - openssl version
+
# Travis provides only 2 cores, so don't overdo the parallelism and waste memory.
before_script:
- - ./configure --with-pydebug PYTHON_FOR_REGEN=python3
+ - ./configure --with-pydebug
- make -j4 regen-all
- changes=`git status --porcelain`
- |
@@ -113,7 +143,7 @@ script:
# Check that all symbols exported by libpython start with "Py" or "_Py"
- make smelly
# `-r -w` implicitly provided through `make buildbottest`.
- - make buildbottest TESTOPTS="-j4 -uall,-cpu"
+ - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then XVFB_RUN=xvfb-run; fi; $XVFB_RUN make buildbottest TESTOPTS="-j4 -uall,-cpu"
notifications:
email: false
diff --git a/.vsts/docs-release.yml b/.vsts/docs-release.yml
deleted file mode 100644
index e90428a42494e0a725e3fd72465fbcc54b758993..0000000000000000000000000000000000000000
--- a/.vsts/docs-release.yml
+++ /dev/null
@@ -1,43 +0,0 @@
-# Current docs for the syntax of this file are at:
-# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
-
-name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
-
-queue:
- name: Hosted Linux Preview
-
-#variables:
-
-steps:
-- checkout: self
- clean: true
- fetchDepth: 5
-
-- script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full
- displayName: 'Install LaTeX'
-
-- task: UsePythonVersion@0
- displayName: 'Use Python 3.6 or later'
- inputs:
- versionSpec: '>=3.6'
-
-- script: python -m pip install sphinx blurb python-docs-theme
- displayName: 'Install build dependencies'
-
-- script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb'
- workingDirectory: '$(build.sourcesDirectory)/Doc'
- displayName: 'Build documentation'
-
-- task: PublishBuildArtifacts@1
- displayName: 'Publish build'
- inputs:
- PathToPublish: '$(build.sourcesDirectory)/Doc/build'
- ArtifactName: build
- publishLocation: Container
-
-- task: PublishBuildArtifacts@1
- displayName: 'Publish dist'
- inputs:
- PathToPublish: '$(build.sourcesDirectory)/Doc/dist'
- ArtifactName: dist
- publishLocation: Container
diff --git a/.vsts/docs.yml b/.vsts/docs.yml
deleted file mode 100644
index 62f6123adb31fef26c51df1e984e7e836b39965e..0000000000000000000000000000000000000000
--- a/.vsts/docs.yml
+++ /dev/null
@@ -1,44 +0,0 @@
-# Current docs for the syntax of this file are at:
-# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
-
-name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
-
-queue:
- name: Hosted Linux Preview
-
-trigger:
- branches:
- include:
- - master
- - 3.7
- - 3.6
- paths:
- include:
- - Doc/*
-
-#variables:
-
-steps:
-- checkout: self
- clean: true
- fetchDepth: 5
-
-- task: UsePythonVersion@0
- displayName: 'Use Python 3.6 or later'
- inputs:
- versionSpec: '>=3.6'
-
-- script: python -m pip install sphinx~=1.6.1 blurb python-docs-theme
- displayName: 'Install build dependencies'
-
-- script: make check suspicious html PYTHON=python
- workingDirectory: '$(build.sourcesDirectory)/Doc'
- displayName: 'Build documentation'
-
-- task: PublishBuildArtifacts@1
- displayName: 'Publish build'
- condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
- inputs:
- PathToPublish: '$(build.sourcesDirectory)/Doc/build'
- ArtifactName: build
- publishLocation: Container
diff --git a/.vsts/linux-buildbot.yml b/.vsts/linux-buildbot.yml
deleted file mode 100644
index 76222d10c48e4cf96cdd23e7b054792165e89b9d..0000000000000000000000000000000000000000
--- a/.vsts/linux-buildbot.yml
+++ /dev/null
@@ -1,64 +0,0 @@
-# Current docs for the syntax of this file are at:
-# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
-
-name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
-
-queue:
- name: Hosted Linux Preview
-
-trigger:
- branches:
- include:
- - master
- - 3.7
- - 3.6
- paths:
- exclude:
- - Doc/*
- - Tools/*
-
-#variables:
-
-
-steps:
-- checkout: self
- clean: true
- fetchDepth: 5
-
-#- template: linux-deps.yml
-
-# See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md
-# For now, we copy/paste the steps
-- script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update
- displayName: 'Update apt-get lists'
-
-- script: >
- sudo apt-get -yq install
- build-essential
- zlib1g-dev
- libbz2-dev
- liblzma-dev
- libncurses5-dev
- libreadline6-dev
- libsqlite3-dev
- libssl-dev
- libgdbm-dev
- tk-dev
- lzma
- lzma-dev
- liblzma-dev
- libffi-dev
- uuid-dev
- displayName: 'Install dependencies'
-
-- script: ./configure --with-pydebug
- displayName: 'Configure CPython (debug)'
-
-- script: make -s -j4
- displayName: 'Build CPython'
-
-- script: make pythoninfo
- displayName: 'Display build info'
-
-- script: make buildbottest TESTOPTS="-j4 -uall,-cpu"
- displayName: 'Tests'
diff --git a/.vsts/linux-coverage.yml b/.vsts/linux-coverage.yml
deleted file mode 100644
index d16d9c9dfa2010edda67dc6bfffe953b233fc015..0000000000000000000000000000000000000000
--- a/.vsts/linux-coverage.yml
+++ /dev/null
@@ -1,70 +0,0 @@
-# Current docs for the syntax of this file are at:
-# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
-
-name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
-
-queue:
- name: Hosted Linux Preview
-
-trigger:
- branches:
- include:
- - master
- - 3.7
- - 3.6
- paths:
- exclude:
- - Doc/*
- - Tools/*
-
-#variables:
-
-steps:
-- checkout: self
- clean: true
- fetchDepth: 5
-
-#- template: linux-deps.yml
-
-# See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md
-# For now, we copy/paste the steps
-- script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update
- displayName: 'Update apt-get lists'
-
-- script: >
- sudo apt-get -yq install
- build-essential
- zlib1g-dev
- libbz2-dev
- liblzma-dev
- libncurses5-dev
- libreadline6-dev
- libsqlite3-dev
- libssl-dev
- libgdbm-dev
- tk-dev
- lzma
- lzma-dev
- liblzma-dev
- libffi-dev
- uuid-dev
- displayName: 'Install dependencies'
-
-
-- script: ./configure --with-pydebug
- displayName: 'Configure CPython (debug)'
-
-- script: make -s -j4
- displayName: 'Build CPython'
-
-- script: ./python -m venv venv && ./venv/bin/python -m pip install -U coverage
- displayName: 'Set up virtual environment'
-
-- script: ./venv/bin/python -m test.pythoninfo
- displayName: 'Display build info'
-
-- script: ./venv/bin/python -m coverage run --pylib -m test -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures
- displayName: 'Tests with coverage'
-
-- script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash)
- displayName: 'Publish code coverage results'
diff --git a/.vsts/linux-deps.yml b/.vsts/linux-deps.yml
deleted file mode 100644
index 540b76ec54c38455ea63d4f80c8e40a63944cc3c..0000000000000000000000000000000000000000
--- a/.vsts/linux-deps.yml
+++ /dev/null
@@ -1,30 +0,0 @@
-# Note: this file is not currently used, but when template support comes to VSTS it
-# will be referenced from the other scripts..
-
-# Current docs for the syntax of this file are at:
-# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
-
-#parameters:
-
-steps:
-- script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update
- displayName: 'Update apt-get lists'
-
-- script: >
- sudo apt-get -yq install
- build-essential
- zlib1g-dev
- libbz2-dev
- liblzma-dev
- libncurses5-dev
- libreadline6-dev
- libsqlite3-dev
- libssl-dev
- libgdbm-dev
- tk-dev
- lzma
- lzma-dev
- liblzma-dev
- libffi-dev
- uuid-dev
- displayName: 'Install dependencies'
diff --git a/.vsts/linux-pr.yml b/.vsts/linux-pr.yml
deleted file mode 100644
index 83df9b436881d726218e8819145fcf1b5c1ab91f..0000000000000000000000000000000000000000
--- a/.vsts/linux-pr.yml
+++ /dev/null
@@ -1,68 +0,0 @@
-# Current docs for the syntax of this file are at:
-# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
-
-name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
-
-queue:
- name: Hosted Linux Preview
-
-trigger:
- branches:
- include:
- - master
- - 3.7
- - 3.6
- paths:
- exclude:
- - Doc/*
- - Tools/*
-
-#variables:
-
-steps:
-- checkout: self
- clean: true
- fetchDepth: 5
-
-#- template: linux-deps.yml
-
-# See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md
-# For now, we copy/paste the steps
-- script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update
- displayName: 'Update apt-get lists'
-
-- script: >
- sudo apt-get -yq install
- build-essential
- zlib1g-dev
- libbz2-dev
- liblzma-dev
- libncurses5-dev
- libreadline6-dev
- libsqlite3-dev
- libssl-dev
- libgdbm-dev
- tk-dev
- lzma
- lzma-dev
- liblzma-dev
- libffi-dev
- uuid-dev
- displayName: 'Install dependencies'
-
-
-- script: ./configure --with-pydebug
- displayName: 'Configure CPython (debug)'
-
-- script: make -s -j4
- displayName: 'Build CPython'
-
-- script: make pythoninfo
- displayName: 'Display build info'
-
-# Run patchcheck and fail if anything is discovered
-- script: ./python Tools/scripts/patchcheck.py --travis true
- displayName: 'Run patchcheck.py'
-
-- script: make buildbottest TESTOPTS="-j4 -uall,-cpu"
- displayName: 'Tests'
diff --git a/.vsts/macos-buildbot.yml b/.vsts/macos-buildbot.yml
deleted file mode 100644
index 8a4f6ba8cb8bff1156c0d3502af6f6b484dde6d0..0000000000000000000000000000000000000000
--- a/.vsts/macos-buildbot.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-# Current docs for the syntax of this file are at:
-# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
-
-name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
-
-queue:
- name: Hosted macOS Preview
-
-trigger:
- branches:
- include:
- - master
- - 3.7
- - 3.6
- paths:
- exclude:
- - Doc/*
- - Tools/*
-
-#variables:
-
-steps:
-- checkout: self
- clean: true
- fetchDepth: 5
-
-- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl
- displayName: 'Configure CPython (debug)'
-
-- script: make -s -j4
- displayName: 'Build CPython'
-
-- script: make pythoninfo
- displayName: 'Display build info'
-
-- script: make buildbottest TESTOPTS="-j4 -uall,-cpu"
- displayName: 'Tests'
diff --git a/.vsts/macos-pr.yml b/.vsts/macos-pr.yml
deleted file mode 100644
index 8a4f6ba8cb8bff1156c0d3502af6f6b484dde6d0..0000000000000000000000000000000000000000
--- a/.vsts/macos-pr.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-# Current docs for the syntax of this file are at:
-# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
-
-name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
-
-queue:
- name: Hosted macOS Preview
-
-trigger:
- branches:
- include:
- - master
- - 3.7
- - 3.6
- paths:
- exclude:
- - Doc/*
- - Tools/*
-
-#variables:
-
-steps:
-- checkout: self
- clean: true
- fetchDepth: 5
-
-- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl
- displayName: 'Configure CPython (debug)'
-
-- script: make -s -j4
- displayName: 'Build CPython'
-
-- script: make pythoninfo
- displayName: 'Display build info'
-
-- script: make buildbottest TESTOPTS="-j4 -uall,-cpu"
- displayName: 'Tests'
diff --git a/.vsts/windows-buildbot.yml b/.vsts/windows-buildbot.yml
deleted file mode 100644
index 5ec4522796cea90439b41dedab32b936ccce05fc..0000000000000000000000000000000000000000
--- a/.vsts/windows-buildbot.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-# Current docs for the syntax of this file are at:
-# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
-
-name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
-
-queue:
- name: Hosted VS2017
- parallel: 2
- matrix:
- amd64:
- buildOpt: -p x64
- outDirSuffix: amd64
- win32:
- buildOpt:
- outDirSuffix: win32
-
-trigger:
- branches:
- include:
- - master
- - 3.7
- - 3.6
- paths:
- exclude:
- - Doc/*
- - Tools/*
-
-variables:
- # Relocate build outputs outside of source directory to make cleaning faster
- Py_IntDir: $(Build.BinariesDirectory)\obj
- # UNDONE: Do not build to a different directory because of broken tests
- Py_OutDir: $(Build.SourcesDirectory)\PCbuild
- EXTERNAL_DIR: $(Build.BinariesDirectory)\externals
-
-steps:
-- checkout: self
- clean: true
- fetchDepth: 5
-
-- script: PCbuild\build.bat -e $(buildOpt)
- displayName: 'Build CPython'
-
-- script: python.bat -m test.pythoninfo
- displayName: 'Display build info'
-
-- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
- displayName: 'Tests'
- env:
- PREFIX: $(Py_OutDir)\$(outDirSuffix)
diff --git a/.vsts/windows-pr.yml b/.vsts/windows-pr.yml
deleted file mode 100644
index 5ec4522796cea90439b41dedab32b936ccce05fc..0000000000000000000000000000000000000000
--- a/.vsts/windows-pr.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-# Current docs for the syntax of this file are at:
-# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md
-
-name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
-
-queue:
- name: Hosted VS2017
- parallel: 2
- matrix:
- amd64:
- buildOpt: -p x64
- outDirSuffix: amd64
- win32:
- buildOpt:
- outDirSuffix: win32
-
-trigger:
- branches:
- include:
- - master
- - 3.7
- - 3.6
- paths:
- exclude:
- - Doc/*
- - Tools/*
-
-variables:
- # Relocate build outputs outside of source directory to make cleaning faster
- Py_IntDir: $(Build.BinariesDirectory)\obj
- # UNDONE: Do not build to a different directory because of broken tests
- Py_OutDir: $(Build.SourcesDirectory)\PCbuild
- EXTERNAL_DIR: $(Build.BinariesDirectory)\externals
-
-steps:
-- checkout: self
- clean: true
- fetchDepth: 5
-
-- script: PCbuild\build.bat -e $(buildOpt)
- displayName: 'Build CPython'
-
-- script: python.bat -m test.pythoninfo
- displayName: 'Display build info'
-
-- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
- displayName: 'Tests'
- env:
- PREFIX: $(Py_OutDir)\$(outDirSuffix)
diff --git a/CODE_OF_CONDUCT.rst b/CODE_OF_CONDUCT.rst
new file mode 100644
index 0000000000000000000000000000000000000000..28de97ced16861d11c49e4cf68e05db8e1acdd7c
--- /dev/null
+++ b/CODE_OF_CONDUCT.rst
@@ -0,0 +1,14 @@
+Code of Conduct
+===============
+
+Please note that all interactions on
+`Python Software Foundation `__-supported
+infrastructure is `covered
+`__
+by the `PSF Code of Conduct `__,
+which includes all infrastructure used in the development of Python itself
+(e.g. mailing lists, issue trackers, GitHub, etc.).
+
+In general this means everyone is expected to be open, considerate, and
+respectful of others no matter what their position is within the project.
+
diff --git a/Doc/README.rst b/Doc/README.rst
index a29d1f3a708a4360802dae3be22bd1ea0688f495..d7bcc5ba7919bbe4d3306fa3fc0910c3fb32f2aa 100644
--- a/Doc/README.rst
+++ b/Doc/README.rst
@@ -33,7 +33,7 @@ To get started on UNIX, you can create a virtual environment with the command ::
make venv
That will install all the tools necessary to build the documentation. Assuming
-the virtual environment was created in the ``env`` directory (the default;
+the virtual environment was created in the ``venv`` directory (the default;
configurable with the VENVDIR variable), you can run the following command to
build the HTML output files::
diff --git a/Doc/bugs.rst b/Doc/bugs.rst
index bc1d10f379cb350bce2c5d8e13b14c58c6bb9edf..109e9eb202d8ea9abad9132237a24fbfe58d08eb 100644
--- a/Doc/bugs.rst
+++ b/Doc/bugs.rst
@@ -68,7 +68,7 @@ taken on the bug.
.. seealso::
- `How to Report Bugs Effectively `_
+ `How to Report Bugs Effectively `_
Article which goes into some detail about how to create a useful bug report.
This describes what kind of information is useful and why it is useful.
diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst
index dc58e152a6b56f3f10f18ed7ee5702f17aa78b97..b41130ede416e39fd4fe6b68c6b6f15e8dfed9b0 100644
--- a/Doc/c-api/arg.rst
+++ b/Doc/c-api/arg.rst
@@ -138,7 +138,7 @@ which disallows mutable objects such as :class:`bytearray`.
attempting any conversion. Raises :exc:`TypeError` if the object is not
a :class:`bytearray` object. The C variable may also be declared as :c:type:`PyObject\*`.
-``u`` (:class:`str`) [Py_UNICODE \*]
+``u`` (:class:`str`) [const Py_UNICODE \*]
Convert a Python Unicode object to a C pointer to a NUL-terminated buffer of
Unicode characters. You must pass the address of a :c:type:`Py_UNICODE`
pointer variable, which will be filled with the pointer to an existing
@@ -155,7 +155,7 @@ which disallows mutable objects such as :class:`bytearray`.
Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
:c:func:`PyUnicode_AsWideCharString`.
-``u#`` (:class:`str`) [Py_UNICODE \*, int]
+``u#`` (:class:`str`) [const Py_UNICODE \*, int]
This variant on ``u`` stores into two C variables, the first one a pointer to a
Unicode data buffer, the second one its length. This variant allows
null code points.
@@ -164,7 +164,7 @@ which disallows mutable objects such as :class:`bytearray`.
Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
:c:func:`PyUnicode_AsWideCharString`.
-``Z`` (:class:`str` or ``None``) [Py_UNICODE \*]
+``Z`` (:class:`str` or ``None``) [const Py_UNICODE \*]
Like ``u``, but the Python object may also be ``None``, in which case the
:c:type:`Py_UNICODE` pointer is set to *NULL*.
@@ -172,7 +172,7 @@ which disallows mutable objects such as :class:`bytearray`.
Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
:c:func:`PyUnicode_AsWideCharString`.
-``Z#`` (:class:`str` or ``None``) [Py_UNICODE \*, int]
+``Z#`` (:class:`str` or ``None``) [const Py_UNICODE \*, int]
Like ``u#``, but the Python object may also be ``None``, in which case the
:c:type:`Py_UNICODE` pointer is set to *NULL*.
@@ -545,43 +545,43 @@ Building values
not within format units such as ``s#``). This can be used to make long format
strings a tad more readable.
- ``s`` (:class:`str` or ``None``) [char \*]
+ ``s`` (:class:`str` or ``None``) [const char \*]
Convert a null-terminated C string to a Python :class:`str` object using ``'utf-8'``
encoding. If the C string pointer is *NULL*, ``None`` is used.
- ``s#`` (:class:`str` or ``None``) [char \*, int]
+ ``s#`` (:class:`str` or ``None``) [const char \*, int]
Convert a C string and its length to a Python :class:`str` object using ``'utf-8'``
encoding. If the C string pointer is *NULL*, the length is ignored and
``None`` is returned.
- ``y`` (:class:`bytes`) [char \*]
+ ``y`` (:class:`bytes`) [const char \*]
This converts a C string to a Python :class:`bytes` object. If the C
string pointer is *NULL*, ``None`` is returned.
- ``y#`` (:class:`bytes`) [char \*, int]
+ ``y#`` (:class:`bytes`) [const char \*, int]
This converts a C string and its lengths to a Python object. If the C
string pointer is *NULL*, ``None`` is returned.
- ``z`` (:class:`str` or ``None``) [char \*]
+ ``z`` (:class:`str` or ``None``) [const char \*]
Same as ``s``.
- ``z#`` (:class:`str` or ``None``) [char \*, int]
+ ``z#`` (:class:`str` or ``None``) [const char \*, int]
Same as ``s#``.
- ``u`` (:class:`str`) [wchar_t \*]
+ ``u`` (:class:`str`) [const wchar_t \*]
Convert a null-terminated :c:type:`wchar_t` buffer of Unicode (UTF-16 or UCS-4)
data to a Python Unicode object. If the Unicode buffer pointer is *NULL*,
``None`` is returned.
- ``u#`` (:class:`str`) [wchar_t \*, int]
+ ``u#`` (:class:`str`) [const wchar_t \*, int]
Convert a Unicode (UTF-16 or UCS-4) data buffer and its length to a Python
Unicode object. If the Unicode buffer pointer is *NULL*, the length is ignored
and ``None`` is returned.
- ``U`` (:class:`str` or ``None``) [char \*]
+ ``U`` (:class:`str` or ``None``) [const char \*]
Same as ``s``.
- ``U#`` (:class:`str` or ``None``) [char \*, int]
+ ``U#`` (:class:`str` or ``None``) [const char \*, int]
Same as ``s#``.
``i`` (:class:`int`) [int]
diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst
index 5a9a46fc67e8995c42cde4ec2bfc0c42733430db..33abb5bb94d9de29fc8c811ae23dc023a91f2535 100644
--- a/Doc/c-api/buffer.rst
+++ b/Doc/c-api/buffer.rst
@@ -198,7 +198,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`.
indicates that no de-referencing should occur (striding in a contiguous
memory block).
- If all suboffsets are negative (i.e. no de-referencing is needed, then
+ If all suboffsets are negative (i.e. no de-referencing is needed), then
this field must be NULL (the default value).
This type of array representation is used by the Python Imaging Library
diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst
index ee42f854ed821d5fe6746618119a93ad88acb736..5b9ebf6b6af5f5f28739480425e3354765bcd1c6 100644
--- a/Doc/c-api/bytes.rst
+++ b/Doc/c-api/bytes.rst
@@ -72,34 +72,34 @@ called with a non-bytes parameter.
| :attr:`%c` | int | A single byte, |
| | | represented as a C int. |
+-------------------+---------------+--------------------------------+
- | :attr:`%d` | int | Exactly equivalent to |
- | | | ``printf("%d")``. |
+ | :attr:`%d` | int | Equivalent to |
+ | | | ``printf("%d")``. [1]_ |
+-------------------+---------------+--------------------------------+
- | :attr:`%u` | unsigned int | Exactly equivalent to |
- | | | ``printf("%u")``. |
+ | :attr:`%u` | unsigned int | Equivalent to |
+ | | | ``printf("%u")``. [1]_ |
+-------------------+---------------+--------------------------------+
- | :attr:`%ld` | long | Exactly equivalent to |
- | | | ``printf("%ld")``. |
+ | :attr:`%ld` | long | Equivalent to |
+ | | | ``printf("%ld")``. [1]_ |
+-------------------+---------------+--------------------------------+
- | :attr:`%lu` | unsigned long | Exactly equivalent to |
- | | | ``printf("%lu")``. |
+ | :attr:`%lu` | unsigned long | Equivalent to |
+ | | | ``printf("%lu")``. [1]_ |
+-------------------+---------------+--------------------------------+
- | :attr:`%zd` | Py_ssize_t | Exactly equivalent to |
- | | | ``printf("%zd")``. |
+ | :attr:`%zd` | Py_ssize_t | Equivalent to |
+ | | | ``printf("%zd")``. [1]_ |
+-------------------+---------------+--------------------------------+
- | :attr:`%zu` | size_t | Exactly equivalent to |
- | | | ``printf("%zu")``. |
+ | :attr:`%zu` | size_t | Equivalent to |
+ | | | ``printf("%zu")``. [1]_ |
+-------------------+---------------+--------------------------------+
- | :attr:`%i` | int | Exactly equivalent to |
- | | | ``printf("%i")``. |
+ | :attr:`%i` | int | Equivalent to |
+ | | | ``printf("%i")``. [1]_ |
+-------------------+---------------+--------------------------------+
- | :attr:`%x` | int | Exactly equivalent to |
- | | | ``printf("%x")``. |
+ | :attr:`%x` | int | Equivalent to |
+ | | | ``printf("%x")``. [1]_ |
+-------------------+---------------+--------------------------------+
- | :attr:`%s` | char\* | A null-terminated C character |
+ | :attr:`%s` | const char\* | A null-terminated C character |
| | | array. |
+-------------------+---------------+--------------------------------+
- | :attr:`%p` | void\* | The hex representation of a C |
+ | :attr:`%p` | const void\* | The hex representation of a C |
| | | pointer. Mostly equivalent to |
| | | ``printf("%p")`` except that |
| | | it is guaranteed to start with |
@@ -111,6 +111,9 @@ called with a non-bytes parameter.
An unrecognized format character causes all the rest of the format string to be
copied as-is to the result object, and any extra arguments discarded.
+ .. [1] For integer specifiers (d, u, ld, lu, zd, zu, i, x): the 0-conversion
+ flag has effect even when a precision is given.
+
.. c:function:: PyObject* PyBytes_FromFormatV(const char *format, va_list vargs)
diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst
index 47dab81891d38ffaff4eed7ce08828ce5ee06a19..9558a4a583cba53245cf484cc07f2039c331ef96 100644
--- a/Doc/c-api/concrete.rst
+++ b/Doc/c-api/concrete.rst
@@ -113,5 +113,5 @@ Other Objects
capsule.rst
gen.rst
coro.rst
+ contextvars.rst
datetime.rst
-
diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst
new file mode 100644
index 0000000000000000000000000000000000000000..c344c8d71ae32d81fe788a244c4a5fdd2e0b66a4
--- /dev/null
+++ b/Doc/c-api/contextvars.rst
@@ -0,0 +1,144 @@
+.. highlightlang:: c
+
+.. _contextvarsobjects:
+
+Context Variables Objects
+-------------------------
+
+.. _contextvarsobjects_pointertype_change:
+.. versionchanged:: 3.7.1
+
+ .. note::
+
+ In Python 3.7.1 the signatures of all context variables
+ C APIs were **changed** to use :c:type:`PyObject` pointers instead
+ of :c:type:`PyContext`, :c:type:`PyContextVar`, and
+ :c:type:`PyContextToken`, e.g.::
+
+ // in 3.7.0:
+ PyContext *PyContext_New(void);
+
+ // in 3.7.1+:
+ PyObject *PyContext_New(void);
+
+ See :issue:`34762` for more details.
+
+
+.. versionadded:: 3.7
+
+This section details the public C API for the :mod:`contextvars` module.
+
+.. c:type:: PyContext
+
+ The C structure used to represent a :class:`contextvars.Context`
+ object.
+
+.. c:type:: PyContextVar
+
+ The C structure used to represent a :class:`contextvars.ContextVar`
+ object.
+
+.. c:type:: PyContextToken
+
+ The C structure used to represent a :class:`contextvars.Token` object.
+
+.. c:var:: PyTypeObject PyContext_Type
+
+ The type object representing the *context* type.
+
+.. c:var:: PyTypeObject PyContextVar_Type
+
+ The type object representing the *context variable* type.
+
+.. c:var:: PyTypeObject PyContextToken_Type
+
+ The type object representing the *context variable token* type.
+
+
+Type-check macros:
+
+.. c:function:: int PyContext_CheckExact(PyObject *o)
+
+ Return true if *o* is of type :c:data:`PyContext_Type`. *o* must not be
+ *NULL*. This function always succeeds.
+
+.. c:function:: int PyContextVar_CheckExact(PyObject *o)
+
+ Return true if *o* is of type :c:data:`PyContextVar_Type`. *o* must not be
+ *NULL*. This function always succeeds.
+
+.. c:function:: int PyContextToken_CheckExact(PyObject *o)
+
+ Return true if *o* is of type :c:data:`PyContextToken_Type`.
+ *o* must not be *NULL*. This function always succeeds.
+
+
+Context object management functions:
+
+.. c:function:: PyObject *PyContext_New(void)
+
+ Create a new empty context object. Returns ``NULL`` if an error
+ has occurred.
+
+.. c:function:: PyObject *PyContext_Copy(PyObject *ctx)
+
+ Create a shallow copy of the passed *ctx* context object.
+ Returns ``NULL`` if an error has occurred.
+
+.. c:function:: PyObject *PyContext_CopyCurrent(void)
+
+ Create a shallow copy of the current thread context.
+ Returns ``NULL`` if an error has occurred.
+
+.. c:function:: int PyContext_Enter(PyObject *ctx)
+
+ Set *ctx* as the current context for the current thread.
+ Returns ``0`` on success, and ``-1`` on error.
+
+.. c:function:: int PyContext_Exit(PyObject *ctx)
+
+ Deactivate the *ctx* context and restore the previous context as the
+ current context for the current thread. Returns ``0`` on success,
+ and ``-1`` on error.
+
+.. c:function:: int PyContext_ClearFreeList()
+
+ Clear the context variable free list. Return the total number of
+ freed items. This function always succeeds.
+
+
+Context variable functions:
+
+.. c:function:: PyObject *PyContextVar_New(const char *name, PyObject *def)
+
+ Create a new ``ContextVar`` object. The *name* parameter is used
+ for introspection and debug purposes. The *def* parameter may optionally
+ specify the default value for the context variable. If an error has
+ occurred, this function returns ``NULL``.
+
+.. c:function:: int PyContextVar_Get(PyObject *var, PyObject *default_value, PyObject **value)
+
+ Get the value of a context variable. Returns ``-1`` if an error has
+ occurred during lookup, and ``0`` if no error occurred, whether or not
+ a value was found.
+
+ If the context variable was found, *value* will be a pointer to it.
+ If the context variable was *not* found, *value* will point to:
+
+ - *default_value*, if not ``NULL``;
+ - the default value of *var*, if not ``NULL``;
+ - ``NULL``
+
+ If the value was found, the function will create a new reference to it.
+
+.. c:function:: PyObject *PyContextVar_Set(PyObject *var, PyObject *value)
+
+ Set the value of *var* to *value* in the current context. Returns a
+ pointer to a :c:type:`PyObject` object, or ``NULL`` if an error
+ has occurred.
+
+.. c:function:: int PyContextVar_Reset(PyObject *var, PyObject *token)
+
+ Reset the state of the *var* context variable to that it was in before
+ :c:func:`PyContextVar_Set` that returned the *token* was called.
+ This function returns ``0`` on success and ``-1`` on error.
diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst
index 9566d9d792000dff0f16a89b1b62e474e933e8a1..c46722d782a2fb7c44c67eef1a04cf903f313a6f 100644
--- a/Doc/c-api/conversion.rst
+++ b/Doc/c-api/conversion.rst
@@ -60,7 +60,7 @@ The following functions provide locale-independent string to number conversions.
The conversion is independent of the current locale.
If ``endptr`` is ``NULL``, convert the whole string. Raise
- ValueError and return ``-1.0`` if the string is not a valid
+ :exc:`ValueError` and return ``-1.0`` if the string is not a valid
representation of a floating-point number.
If endptr is not ``NULL``, convert as much of the string as
diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst
index 305e990368c29314c2c33fa726bc6fd3999a1dc3..78724619ea3c52267d36ac6d8740be4bb983bb89 100644
--- a/Doc/c-api/datetime.rst
+++ b/Doc/c-api/datetime.rst
@@ -13,6 +13,16 @@ the module initialisation function. The macro puts a pointer to a C structure
into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following
macros.
+Macro for access to the UTC singleton:
+
+.. c:var:: PyObject* PyDateTime_TimeZone_UTC
+
+ Returns the time zone singleton representing UTC, the same object as
+ :attr:`datetime.timezone.utc`.
+
+ .. versionadded:: 3.7
+
+
Type-check macros:
.. c:function:: int PyDate_Check(PyObject *ob)
@@ -79,27 +89,41 @@ Macros to create objects:
.. c:function:: PyObject* PyDate_FromDate(int year, int month, int day)
- Return a ``datetime.date`` object with the specified year, month and day.
+ Return a :class:`datetime.date` object with the specified year, month and day.
.. c:function:: PyObject* PyDateTime_FromDateAndTime(int year, int month, int day, int hour, int minute, int second, int usecond)
- Return a ``datetime.datetime`` object with the specified year, month, day, hour,
+ Return a :class:`datetime.datetime` object with the specified year, month, day, hour,
minute, second and microsecond.
.. c:function:: PyObject* PyTime_FromTime(int hour, int minute, int second, int usecond)
- Return a ``datetime.time`` object with the specified hour, minute, second and
+ Return a :class:`datetime.time` object with the specified hour, minute, second and
microsecond.
.. c:function:: PyObject* PyDelta_FromDSU(int days, int seconds, int useconds)
- Return a ``datetime.timedelta`` object representing the given number of days,
- seconds and microseconds. Normalization is performed so that the resulting
- number of microseconds and seconds lie in the ranges documented for
- ``datetime.timedelta`` objects.
+ Return a :class:`datetime.timedelta` object representing the given number
+ of days, seconds and microseconds. Normalization is performed so that the
+ resulting number of microseconds and seconds lie in the ranges documented for
+ :class:`datetime.timedelta` objects.
+
+.. c:function:: PyObject* PyTimeZone_FromOffset(PyDateTime_DeltaType* offset)
+
+ Return a :class:`datetime.timezone` object with an unnamed fixed offset
+ represented by the *offset* argument.
+
+ .. versionadded:: 3.7
+
+.. c:function:: PyObject* PyTimeZone_FromOffsetAndName(PyDateTime_DeltaType* offset, PyUnicode* name)
+
+ Return a :class:`datetime.timezone` object with a fixed offset represented
+ by the *offset* argument and with tzname *name*.
+
+ .. versionadded:: 3.7
Macros to extract fields from date objects. The argument must be an instance of
@@ -199,11 +223,11 @@ Macros for the convenience of modules implementing the DB API:
.. c:function:: PyObject* PyDateTime_FromTimestamp(PyObject *args)
- Create and return a new ``datetime.datetime`` object given an argument tuple
- suitable for passing to ``datetime.datetime.fromtimestamp()``.
+ Create and return a new :class:`datetime.datetime` object given an argument
+ tuple suitable for passing to :meth:`datetime.datetime.fromtimestamp()`.
.. c:function:: PyObject* PyDate_FromTimestamp(PyObject *args)
- Create and return a new ``datetime.date`` object given an argument tuple
- suitable for passing to ``datetime.date.fromtimestamp()``.
+ Create and return a new :class:`datetime.date` object given an argument
+ tuple suitable for passing to :meth:`datetime.date.fromtimestamp()`.
diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst
index cfa5e13b9feef8a21d16c6178be7b60771931a7b..b7225faf408ca0abed07d4f5445c7b916944b0d1 100644
--- a/Doc/c-api/dict.rst
+++ b/Doc/c-api/dict.rst
@@ -72,7 +72,7 @@ Dictionary Objects
.. index:: single: PyUnicode_FromString()
Insert *value* into the dictionary *p* using *key* as a key. *key* should
- be a :c:type:`char\*`. The key object is created using
+ be a :c:type:`const char\*`. The key object is created using
``PyUnicode_FromString(key)``. Return ``0`` on success or ``-1`` on
failure.
@@ -107,7 +107,7 @@ Dictionary Objects
.. c:function:: PyObject* PyDict_GetItemString(PyObject *p, const char *key)
This is the same as :c:func:`PyDict_GetItem`, but *key* is specified as a
- :c:type:`char\*`, rather than a :c:type:`PyObject\*`.
+ :c:type:`const char\*`, rather than a :c:type:`PyObject\*`.
.. c:function:: PyObject* PyDict_SetDefault(PyObject *p, PyObject *key, PyObject *default)
diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst
index 2bc1bd876a2fe271086ebcd97daa9001794c2fd9..dd1e026cb0716aa4af976577eb16abd84ce68aca 100644
--- a/Doc/c-api/exceptions.rst
+++ b/Doc/c-api/exceptions.rst
@@ -186,34 +186,42 @@ NULL pointer for use in a ``return`` statement.
then it constructs a tuple object whose first item is the *ierr* value and whose
second item is the corresponding error message (gotten from
:c:func:`FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError,
- object)``. This function always returns *NULL*. Availability: Windows.
+ object)``. This function always returns *NULL*.
+
+ .. availability:: Windows.
.. c:function:: PyObject* PyErr_SetExcFromWindowsErr(PyObject *type, int ierr)
Similar to :c:func:`PyErr_SetFromWindowsErr`, with an additional parameter
- specifying the exception type to be raised. Availability: Windows.
+ specifying the exception type to be raised.
+
+ .. availability:: Windows.
.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename)
Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the
filename is given as a C string. *filename* is decoded from the filesystem
- encoding (:func:`os.fsdecode`). Availability: Windows.
+ encoding (:func:`os.fsdecode`).
+
+ .. availability:: Windows.
.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename)
Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, with an
additional parameter specifying the exception type to be raised.
- Availability: Windows.
+
+ .. availability:: Windows.
.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObjects(PyObject *type, int ierr, PyObject *filename, PyObject *filename2)
Similar to :c:func:`PyErr_SetExcFromWindowsErrWithFilenameObject`,
but accepts a second filename object.
- Availability: Windows.
+
+ .. availability:: Windows.
.. versionadded:: 3.4
@@ -221,7 +229,9 @@ NULL pointer for use in a ``return`` statement.
.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilename(PyObject *type, int ierr, const char *filename)
Similar to :c:func:`PyErr_SetFromWindowsErrWithFilename`, with an additional
- parameter specifying the exception type to be raised. Availability: Windows.
+ parameter specifying the exception type to be raised.
+
+ .. availability:: Windows.
.. c:function:: PyObject* PyErr_SetImportError(PyObject *msg, PyObject *name, PyObject *path)
diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst
index 7f54b6a9cff8c5b97e725f4234d3d478e5a327f3..472cd93ec3f7e196b2bd77016edb73237d903738 100644
--- a/Doc/c-api/gcsupport.rst
+++ b/Doc/c-api/gcsupport.rst
@@ -66,6 +66,9 @@ Constructors for container types must conform to two rules:
A macro version of :c:func:`PyObject_GC_Track`. It should not be used for
extension modules.
+ .. deprecated:: 3.6
+ This macro is removed from Python 3.8.
+
Similarly, the deallocator for the object must conform to a similar pair of
rules:
@@ -95,6 +98,9 @@ rules:
A macro version of :c:func:`PyObject_GC_UnTrack`. It should not be used for
extension modules.
+ .. deprecated:: 3.6
+ This macro is removed from Python 3.8.
+
The :c:member:`~PyTypeObject.tp_traverse` handler accepts a function parameter of this type:
diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst
index 5273633667e9e2a2cc171a9623278af31805cf64..8cdc256e7c9e0e09c269a7549866a9620c9ef21b 100644
--- a/Doc/c-api/import.rst
+++ b/Doc/c-api/import.rst
@@ -204,6 +204,13 @@ Importing Modules
Return the dictionary used for the module administration (a.k.a.
``sys.modules``). Note that this is a per-interpreter variable.
+.. c:function:: PyObject* PyImport_GetModule(PyObject *name)
+
+ Return the already imported module with the given name. If the
+ module has not been imported yet then returns NULL but does not set
+ an error. Returns NULL and sets an error if the lookup failed.
+
+ .. versionadded:: 3.7
.. c:function:: PyObject* PyImport_GetImporter(PyObject *path)
@@ -231,11 +238,6 @@ Importing Modules
Finalize the import mechanism. For internal use only.
-.. c:function:: PyObject* _PyImport_FindExtension(char *, char *)
-
- For internal use only.
-
-
.. c:function:: int PyImport_ImportFrozenModuleObject(PyObject *name)
Load a frozen module named *name*. Return ``1`` for success, ``0`` if the
@@ -266,8 +268,8 @@ Importing Modules
is::
struct _frozen {
- char *name;
- unsigned char *code;
+ const char *name;
+ const unsigned char *code;
int size;
};
@@ -300,7 +302,7 @@ Importing Modules
The structure is defined in :file:`Include/import.h` as::
struct _inittab {
- char *name; /* ASCII encoded string */
+ const char *name; /* ASCII encoded string */
PyObject* (*initfunc)(void);
};
diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst
index f601476211411a19dafe7896958cc724c04e2b78..50998be5747ba47af24e3cb4c84f47054bfdd7b0 100644
--- a/Doc/c-api/init.rst
+++ b/Doc/c-api/init.rst
@@ -7,6 +7,216 @@
Initialization, Finalization, and Threads
*****************************************
+.. _pre-init-safe:
+
+Before Python Initialization
+============================
+
+In an application embedding Python, the :c:func:`Py_Initialize` function must
+be called before using any other Python/C API functions; with the exception of
+a few functions and the :ref:`global configuration variables
+`.
+
+The following functions can be safely called before Python is initialized:
+
+* Configuration functions:
+
+ * :c:func:`PyImport_AppendInittab`
+ * :c:func:`PyImport_ExtendInittab`
+ * :c:func:`PyInitFrozenExtensions`
+ * :c:func:`PyMem_SetAllocator`
+ * :c:func:`PyMem_SetupDebugHooks`
+ * :c:func:`PyObject_SetArenaAllocator`
+ * :c:func:`Py_SetPath`
+ * :c:func:`Py_SetProgramName`
+ * :c:func:`Py_SetPythonHome`
+ * :c:func:`Py_SetStandardStreamEncoding`
+ * :c:func:`PySys_AddWarnOption`
+ * :c:func:`PySys_AddXOption`
+ * :c:func:`PySys_ResetWarnOptions`
+
+* Informative functions:
+
+ * :c:func:`PyMem_GetAllocator`
+ * :c:func:`PyObject_GetArenaAllocator`
+ * :c:func:`Py_GetBuildInfo`
+ * :c:func:`Py_GetCompiler`
+ * :c:func:`Py_GetCopyright`
+ * :c:func:`Py_GetPlatform`
+ * :c:func:`Py_GetVersion`
+
+* Utilities:
+
+ * :c:func:`Py_DecodeLocale`
+
+* Memory allocators:
+
+ * :c:func:`PyMem_RawMalloc`
+ * :c:func:`PyMem_RawRealloc`
+ * :c:func:`PyMem_RawCalloc`
+ * :c:func:`PyMem_RawFree`
+
+.. note::
+
+ The following functions **should not be called** before
+ :c:func:`Py_Initialize`: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`,
+ :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`,
+ :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome`,
+ :c:func:`Py_GetProgramName` and :c:func:`PyEval_InitThreads`.
+
+
+.. _global-conf-vars:
+
+Global configuration variables
+==============================
+
+Python has variables for the global configuration to control different features
+and options. By default, these flags are controlled by :ref:`command line
+options `.
+
+When a flag is set by an option, the value of the flag is the number of times
+that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag`
+to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
+
+.. c:var:: Py_BytesWarningFlag
+
+ Issue a warning when comparing :class:`bytes` or :class:`bytearray` with
+ :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater
+ or equal to ``2``.
+
+ Set by the :option:`-b` option.
+
+.. c:var:: Py_DebugFlag
+
+ Turn on parser debugging output (for expert only, depending on compilation
+ options).
+
+ Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment
+ variable.
+
+.. c:var:: Py_DontWriteBytecodeFlag
+
+ If set to non-zero, Python won't try to write ``.pyc`` files on the
+ import of source modules.
+
+ Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE`
+ environment variable.
+
+.. c:var:: Py_FrozenFlag
+
+ Suppress error messages when calculating the module search path in
+ :c:func:`Py_GetPath`.
+
+ Private flag used by ``_freeze_importlib`` and ``frozenmain`` programs.
+
+.. c:var:: Py_HashRandomizationFlag
+
+ Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to
+ a non-empty string.
+
+ If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment
+ variable to initialize the secret hash seed.
+
+.. c:var:: Py_IgnoreEnvironmentFlag
+
+ Ignore all :envvar:`PYTHON*` environment variables, e.g.
+ :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set.
+
+ Set by the :option:`-E` and :option:`-I` options.
+
+.. c:var:: Py_InspectFlag
+
+ When a script is passed as first argument or the :option:`-c` option is used,
+ enter interactive mode after executing the script or the command, even when
+ :data:`sys.stdin` does not appear to be a terminal.
+
+ Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment
+ variable.
+
+.. c:var:: Py_InteractiveFlag
+
+ Set by the :option:`-i` option.
+
+.. c:var:: Py_IsolatedFlag
+
+ Run Python in isolated mode. In isolated mode :data:`sys.path` contains
+ neither the script's directory nor the user's site-packages directory.
+
+ Set by the :option:`-I` option.
+
+ .. versionadded:: 3.4
+
+.. c:var:: Py_LegacyWindowsFSEncodingFlag
+
+ If the flag is non-zero, use the ``mbcs`` encoding instead of the UTF-8
+ encoding for the filesystem encoding.
+
+ Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment
+ variable is set to a non-empty string.
+
+ See :pep:`529` for more details.
+
+ .. availability:: Windows.
+
+.. c:var:: Py_LegacyWindowsStdioFlag
+
+ If the flag is non-zero, use :class:`io.FileIO` instead of
+ :class:`WindowsConsoleIO` for :mod:`sys` standard streams.
+
+ Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment
+ variable is set to a non-empty string.
+
+ See :pep:`528` for more details.
+
+ .. availability:: Windows.
+
+.. c:var:: Py_NoSiteFlag
+
+ Disable the import of the module :mod:`site` and the site-dependent
+ manipulations of :data:`sys.path` that it entails. Also disable these
+ manipulations if :mod:`site` is explicitly imported later (call
+ :func:`site.main` if you want them to be triggered).
+
+ Set by the :option:`-S` option.
+
+.. c:var:: Py_NoUserSiteDirectory
+
+ Don't add the :data:`user site-packages directory ` to
+ :data:`sys.path`.
+
+ Set by the :option:`-s` and :option:`-I` options, and the
+ :envvar:`PYTHONNOUSERSITE` environment variable.
+
+.. c:var:: Py_OptimizeFlag
+
+ Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment
+ variable.
+
+.. c:var:: Py_QuietFlag
+
+ Don't display the copyright and version messages even in interactive mode.
+
+ Set by the :option:`-q` option.
+
+ .. versionadded:: 3.2
+
+.. c:var:: Py_UnbufferedStdioFlag
+
+ Force the stdout and stderr streams to be unbuffered.
+
+ Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED`
+ environment variable.
+
+.. c:var:: Py_VerboseFlag
+
+ Print a message each time a module is initialized, showing the place
+ (filename or built-in module) from which it is loaded. If greater or equal
+ to ``2``, print a message for each file that is checked for when
+ searching for a module. Also provides information on module cleanup at exit.
+
+ Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment
+ variable.
+
Initializing and finalizing the interpreter
===========================================
@@ -27,9 +237,11 @@ Initializing and finalizing the interpreter
single: PySys_SetArgvEx()
single: Py_FinalizeEx()
- Initialize the Python interpreter. In an application embedding Python, this
- should be called before using any other Python/C API functions; with the
- exception of :c:func:`Py_SetProgramName`, :c:func:`Py_SetPythonHome` and :c:func:`Py_SetPath`. This initializes
+ Initialize the Python interpreter. In an application embedding Python,
+ this should be called before using any other Python/C API functions; see
+ :ref:`Before Python Initialization ` for the few exceptions.
+
+ This initializes
the table of loaded modules (``sys.modules``), and creates the fundamental
modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. It also initializes
the module search path (``sys.path``). It does not set ``sys.argv``; use
@@ -129,7 +341,7 @@ Process-wide parameters
.. versionadded:: 3.4
-.. c:function:: void Py_SetProgramName(wchar_t *name)
+.. c:function:: void Py_SetProgramName(const wchar_t *name)
.. index::
single: Py_Initialize()
@@ -396,7 +608,7 @@ Process-wide parameters
.. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`.
-.. c:function:: void Py_SetPythonHome(wchar_t *home)
+.. c:function:: void Py_SetPythonHome(const wchar_t *home)
Set the default "home" directory, that is, the location of the standard
Python libraries. See :envvar:`PYTHONHOME` for the meaning of the
@@ -478,15 +690,14 @@ This is so common that a pair of macros exists to simplify it::
The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a
hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the
-block. These two macros are still available when Python is compiled without
-thread support (they simply have an empty expansion).
+block.
-When thread support is enabled, the block above expands to the following code::
+The block above expands to the following code::
PyThreadState *_save;
_save = PyEval_SaveThread();
- ...Do some blocking I/O operation...
+ ... Do some blocking I/O operation ...
PyEval_RestoreThread(_save);
.. index::
@@ -564,7 +775,7 @@ Additionally, when extending or embedding Python, calling :c:func:`fork`
directly rather than through :func:`os.fork` (and returning to or calling
into Python) may result in a deadlock by one of Python's internal locks
being held by a thread that is defunct after the fork.
-:c:func:`PyOS_AfterFork` tries to reset the necessary locks, but is not
+:c:func:`PyOS_AfterFork_Child` tries to reset the necessary locks, but is not
always able to.
@@ -609,36 +820,24 @@ code, or when embedding the Python interpreter:
This is a no-op when called for a second time.
+ .. versionchanged:: 3.7
+ This function is now called by :c:func:`Py_Initialize()`, so you don't
+ have to call it yourself anymore.
+
.. versionchanged:: 3.2
This function cannot be called before :c:func:`Py_Initialize()` anymore.
.. index:: module: _thread
- .. note::
-
- When only the main thread exists, no GIL operations are needed. This is a
- common situation (most Python programs do not use threads), and the lock
- operations slow the interpreter down a bit. Therefore, the lock is not
- created initially. This situation is equivalent to having acquired the lock:
- when there is only a single thread, all object accesses are safe. Therefore,
- when this function initializes the global interpreter lock, it also acquires
- it. Before the Python :mod:`_thread` module creates a new thread, knowing
- that either it has the lock or the lock hasn't been created yet, it calls
- :c:func:`PyEval_InitThreads`. When this call returns, it is guaranteed that
- the lock has been created and that the calling thread has acquired it.
-
- It is **not** safe to call this function when it is unknown which thread (if
- any) currently has the global interpreter lock.
-
- This function is not available when thread support is disabled at compile time.
-
.. c:function:: int PyEval_ThreadsInitialized()
Returns a non-zero value if :c:func:`PyEval_InitThreads` has been called. This
function can be called without holding the GIL, and therefore can be used to
- avoid calls to the locking API when running single-threaded. This function is
- not available when thread support is disabled at compile time.
+ avoid calls to the locking API when running single-threaded.
+
+ .. versionchanged:: 3.7
+ The :term:`GIL` is now initialized by :c:func:`Py_Initialize()`.
.. c:function:: PyThreadState* PyEval_SaveThread()
@@ -646,8 +845,7 @@ code, or when embedding the Python interpreter:
Release the global interpreter lock (if it has been created and thread
support is enabled) and reset the thread state to *NULL*, returning the
previous thread state (which is not *NULL*). If the lock has been created,
- the current thread must have acquired it. (This function is available even
- when thread support is disabled at compile time.)
+ the current thread must have acquired it.
.. c:function:: void PyEval_RestoreThread(PyThreadState *tstate)
@@ -655,8 +853,7 @@ code, or when embedding the Python interpreter:
Acquire the global interpreter lock (if it has been created and thread
support is enabled) and set the thread state to *tstate*, which must not be
*NULL*. If the lock has been created, the current thread must not have
- acquired it, otherwise deadlock ensues. (This function is available even
- when thread support is disabled at compile time.)
+ acquired it, otherwise deadlock ensues.
.. c:function:: PyThreadState* PyThreadState_Get()
@@ -675,9 +872,9 @@ code, or when embedding the Python interpreter:
.. c:function:: void PyEval_ReInitThreads()
- This function is called from :c:func:`PyOS_AfterFork` to ensure that newly
- created child processes don't hold locks referring to threads which
- are not running in the child process.
+ This function is called from :c:func:`PyOS_AfterFork_Child` to ensure
+ that newly created child processes don't hold locks referring to threads
+ which are not running in the child process.
The following functions use thread-local storage, and are not compatible
@@ -748,7 +945,7 @@ example usage in the Python source distribution.
This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``.
Note that it contains an opening brace; it must be matched with a following
:c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this
- macro. It is a no-op when thread support is disabled at compile time.
+ macro.
.. c:macro:: Py_END_ALLOW_THREADS
@@ -756,29 +953,29 @@ example usage in the Python source distribution.
This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains
a closing brace; it must be matched with an earlier
:c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of
- this macro. It is a no-op when thread support is disabled at compile time.
+ this macro.
.. c:macro:: Py_BLOCK_THREADS
This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to
- :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. It is a no-op when
- thread support is disabled at compile time.
+ :c:macro:`Py_END_ALLOW_THREADS` without the closing brace.
.. c:macro:: Py_UNBLOCK_THREADS
This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to
:c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable
- declaration. It is a no-op when thread support is disabled at compile time.
+ declaration.
Low-level API
-------------
-All of the following functions are only available when thread support is enabled
-at compile time, and must be called only when the global interpreter lock has
-been created.
+All of the following functions must be called after :c:func:`Py_Initialize`.
+
+.. versionchanged:: 3.7
+ :c:func:`Py_Initialize()` now initializes the :term:`GIL`.
.. c:function:: PyInterpreterState* PyInterpreterState_New()
@@ -821,6 +1018,14 @@ been created.
:c:func:`PyThreadState_Clear`.
+.. c:function:: PY_INT64_T PyInterpreterState_GetID(PyInterpreterState *interp)
+
+ Return the interpreter's unique ID. If there was any error in doing
+ so then ``-1`` is returned and an error is set.
+
+ .. versionadded:: 3.7
+
+
.. c:function:: PyObject* PyThreadState_GetDict()
Return a dictionary in which extensions can store thread-specific state
@@ -830,7 +1035,7 @@ been created.
the caller should assume no current thread state is available.
-.. c:function:: int PyThreadState_SetAsyncExc(long id, PyObject *exc)
+.. c:function:: int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc)
Asynchronously raise an exception in a thread. The *id* argument is the thread
id of the target thread; *exc* is the exception object to be raised. This
@@ -840,6 +1045,9 @@ been created.
zero if the thread id isn't found. If *exc* is :const:`NULL`, the pending
exception (if any) for the thread is cleared. This raises no exceptions.
+ .. versionchanged:: 3.7
+ The type of the *id* parameter changed from :c:type:`long` to
+ :c:type:`unsigned long`.
.. c:function:: void PyEval_AcquireThread(PyThreadState *tstate)
@@ -848,8 +1056,7 @@ been created.
If this thread already has the lock, deadlock ensues.
:c:func:`PyEval_RestoreThread` is a higher-level function which is always
- available (even when thread support isn't enabled or when threads have
- not been initialized).
+ available (even when threads have not been initialized).
.. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate)
@@ -861,8 +1068,7 @@ been created.
reported.
:c:func:`PyEval_SaveThread` is a higher-level function which is always
- available (even when thread support isn't enabled or when threads have
- not been initialized).
+ available (even when threads have not been initialized).
.. c:function:: void PyEval_AcquireLock()
@@ -1057,8 +1263,8 @@ Python-level trace functions in previous versions.
registration function as *obj*, *frame* is the frame object to which the event
pertains, *what* is one of the constants :const:`PyTrace_CALL`,
:const:`PyTrace_EXCEPTION`, :const:`PyTrace_LINE`, :const:`PyTrace_RETURN`,
- :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION`, or
- :const:`PyTrace_C_RETURN`, and *arg* depends on the value of *what*:
+ :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION`, :const:`PyTrace_C_RETURN`,
+ or :const:`PyTrace_OPCODE`, and *arg* depends on the value of *what*:
+------------------------------+--------------------------------------+
| Value of *what* | Meaning of *arg* |
@@ -1079,7 +1285,8 @@ Python-level trace functions in previous versions.
+------------------------------+--------------------------------------+
| :const:`PyTrace_C_RETURN` | Function object being called. |
+------------------------------+--------------------------------------+
-
+ | :const:`PyTrace_OPCODE` | Always :c:data:`Py_None`. |
+ +------------------------------+--------------------------------------+
.. c:var:: int PyTrace_CALL
@@ -1103,8 +1310,9 @@ Python-level trace functions in previous versions.
.. c:var:: int PyTrace_LINE
- The value passed as the *what* parameter to a trace function (but not a
- profiling function) when a line-number event is being reported.
+ The value passed as the *what* parameter to a :c:type:`Py_tracefunc` function
+ (but not a profiling function) when a line-number event is being reported.
+ It may be disabled for a frame by setting :attr:`f_trace_lines` to *0* on that frame.
.. c:var:: int PyTrace_RETURN
@@ -1131,6 +1339,14 @@ Python-level trace functions in previous versions.
function has returned.
+.. c:var:: int PyTrace_OPCODE
+
+ The value for the *what* parameter to :c:type:`Py_tracefunc` functions (but not
+ profiling functions) when a new opcode is about to be executed. This event is
+ not emitted by default: it must be explicitly requested by setting
+ :attr:`f_trace_opcodes` to *1* on the frame.
+
+
.. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj)
Set the profiler function to *func*. The *obj* parameter is passed to the
@@ -1138,59 +1354,17 @@ Python-level trace functions in previous versions.
the profile function needs to maintain state, using a different value for *obj*
for each thread provides a convenient and thread-safe place to store it. The
profile function is called for all monitored events except :const:`PyTrace_LINE`
- and :const:`PyTrace_EXCEPTION`.
+ :const:`PyTrace_OPCODE` and :const:`PyTrace_EXCEPTION`.
.. c:function:: void PyEval_SetTrace(Py_tracefunc func, PyObject *obj)
Set the tracing function to *func*. This is similar to
:c:func:`PyEval_SetProfile`, except the tracing function does receive line-number
- events and does not receive any event related to C function objects being called. Any
- trace function registered using :c:func:`PyEval_SetTrace` will not receive
- :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION` or :const:`PyTrace_C_RETURN`
- as a value for the *what* parameter.
-
-
-.. c:function:: PyObject* PyEval_GetCallStats(PyObject *self)
-
- Return a tuple of function call counts. There are constants defined for the
- positions within the tuple:
-
- +-------------------------------+-------+
- | Name | Value |
- +===============================+=======+
- | :const:`PCALL_ALL` | 0 |
- +-------------------------------+-------+
- | :const:`PCALL_FUNCTION` | 1 |
- +-------------------------------+-------+
- | :const:`PCALL_FAST_FUNCTION` | 2 |
- +-------------------------------+-------+
- | :const:`PCALL_FASTER_FUNCTION`| 3 |
- +-------------------------------+-------+
- | :const:`PCALL_METHOD` | 4 |
- +-------------------------------+-------+
- | :const:`PCALL_BOUND_METHOD` | 5 |
- +-------------------------------+-------+
- | :const:`PCALL_CFUNCTION` | 6 |
- +-------------------------------+-------+
- | :const:`PCALL_TYPE` | 7 |
- +-------------------------------+-------+
- | :const:`PCALL_GENERATOR` | 8 |
- +-------------------------------+-------+
- | :const:`PCALL_OTHER` | 9 |
- +-------------------------------+-------+
- | :const:`PCALL_POP` | 10 |
- +-------------------------------+-------+
-
- :const:`PCALL_FAST_FUNCTION` means no argument tuple needs to be created.
- :const:`PCALL_FASTER_FUNCTION` means that the fast-path frame setup code is used.
-
- If there is a method call where the call can be optimized by changing
- the argument tuple and calling the function directly, it gets recorded
- twice.
-
- This function is only present if Python is compiled with :const:`CALL_PROFILE`
- defined.
+ events and per-opcode events, but does not receive any event related to C function
+ objects being called. Any trace function registered using :c:func:`PyEval_SetTrace`
+ will not receive :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION` or
+ :const:`PyTrace_C_RETURN` as a value for the *what* parameter.
.. _advanced-debugging:
@@ -1225,3 +1399,160 @@ These functions are only intended to be used by advanced debugging tools.
Return the next thread state object after *tstate* from the list of all such
objects belonging to the same :c:type:`PyInterpreterState` object.
+
+.. _thread-local-storage:
+
+Thread Local Storage Support
+============================
+
+.. sectionauthor:: Masayuki Yamamoto
+
+The Python interpreter provides low-level support for thread-local storage
+(TLS) which wraps the underlying native TLS implementation to support the
+Python-level thread local storage API (:class:`threading.local`). The
+CPython C level APIs are similar to those offered by pthreads and Windows:
+use a thread key and functions to associate a :c:type:`void\*` value per
+thread.
+
+The GIL does *not* need to be held when calling these functions; they supply
+their own locking.
+
+Note that :file:`Python.h` does not include the declaration of the TLS APIs,
+you need to include :file:`pythread.h` to use thread-local storage.
+
+.. note::
+ None of these API functions handle memory management on behalf of the
+ :c:type:`void\*` values. You need to allocate and deallocate them yourself.
+ If the :c:type:`void\*` values happen to be :c:type:`PyObject\*`, these
+ functions don't do refcount operations on them either.
+
+.. _thread-specific-storage-api:
+
+Thread Specific Storage (TSS) API
+---------------------------------
+
+TSS API is introduced to supersede the use of the existing TLS API within the
+CPython interpreter. This API uses a new type :c:type:`Py_tss_t` instead of
+:c:type:`int` to represent thread keys.
+
+.. versionadded:: 3.7
+
+.. seealso:: "A New C-API for Thread-Local Storage in CPython" (:pep:`539`)
+
+
+.. c:type:: Py_tss_t
+
+ This data structure represents the state of a thread key, the definition of
+ which may depend on the underlying TLS implementation, and it has an
+ internal field representing the key's initialization state. There are no
+ public members in this structure.
+
+ When :ref:`Py_LIMITED_API ` is not defined, static allocation of
+ this type by :c:macro:`Py_tss_NEEDS_INIT` is allowed.
+
+
+.. c:macro:: Py_tss_NEEDS_INIT
+
+ This macro expands to the initializer for :c:type:`Py_tss_t` variables.
+ Note that this macro won't be defined with :ref:`Py_LIMITED_API `.
+
+
+Dynamic Allocation
+~~~~~~~~~~~~~~~~~~
+
+Dynamic allocation of the :c:type:`Py_tss_t`, required in extension modules
+built with :ref:`Py_LIMITED_API `, where static allocation of this type
+is not possible due to its implementation being opaque at build time.
+
+
+.. c:function:: Py_tss_t* PyThread_tss_alloc()
+
+ Return a value which is the same state as a value initialized with
+ :c:macro:`Py_tss_NEEDS_INIT`, or *NULL* in the case of dynamic allocation
+ failure.
+
+
+.. c:function:: void PyThread_tss_free(Py_tss_t *key)
+
+ Free the given *key* allocated by :c:func:`PyThread_tss_alloc`, after
+ first calling :c:func:`PyThread_tss_delete` to ensure any associated
+ thread locals have been unassigned. This is a no-op if the *key*
+ argument is `NULL`.
+
+ .. note::
+ A freed key becomes a dangling pointer, you should reset the key to
+ `NULL`.
+
+
+Methods
+~~~~~~~
+
+The parameter *key* of these functions must not be *NULL*. Moreover, the
+behaviors of :c:func:`PyThread_tss_set` and :c:func:`PyThread_tss_get` are
+undefined if the given :c:type:`Py_tss_t` has not been initialized by
+:c:func:`PyThread_tss_create`.
+
+
+.. c:function:: int PyThread_tss_is_created(Py_tss_t *key)
+
+ Return a non-zero value if the given :c:type:`Py_tss_t` has been initialized
+ by :c:func:`PyThread_tss_create`.
+
+
+.. c:function:: int PyThread_tss_create(Py_tss_t *key)
+
+ Return a zero value on successful initialization of a TSS key. The behavior
+ is undefined if the value pointed to by the *key* argument is not
+ initialized by :c:macro:`Py_tss_NEEDS_INIT`. This function can be called
+ repeatedly on the same key -- calling it on an already initialized key is a
+ no-op and immediately returns success.
+
+
+.. c:function:: void PyThread_tss_delete(Py_tss_t *key)
+
+ Destroy a TSS key to forget the values associated with the key across all
+ threads, and change the key's initialization state to uninitialized. A
+ destroyed key is able to be initialized again by
+ :c:func:`PyThread_tss_create`. This function can be called repeatedly on
+ the same key -- calling it on an already destroyed key is a no-op.
+
+
+.. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value)
+
+ Return a zero value to indicate successfully associating a :c:type:`void\*`
+ value with a TSS key in the current thread. Each thread has a distinct
+ mapping of the key to a :c:type:`void\*` value.
+
+
+.. c:function:: void* PyThread_tss_get(Py_tss_t *key)
+
+ Return the :c:type:`void\*` value associated with a TSS key in the current
+ thread. This returns *NULL* if no value is associated with the key in the
+ current thread.
+
+
+.. _thread-local-storage-api:
+
+Thread Local Storage (TLS) API
+------------------------------
+
+.. deprecated:: 3.7
+ This API is superseded by
+ :ref:`Thread Specific Storage (TSS) API `.
+
+.. note::
+ This version of the API does not support platforms where the native TLS key
+ is defined in a way that cannot be safely cast to ``int``. On such platforms,
+ :c:func:`PyThread_create_key` will return immediately with a failure status,
+ and the other TLS functions will all be no-ops on such platforms.
+
+Due to the compatibility problem noted above, this version of the API should not
+be used in new code.
+
+.. c:function:: int PyThread_create_key()
+.. c:function:: void PyThread_delete_key(int key)
+.. c:function:: int PyThread_set_key_value(int key, void *value)
+.. c:function:: void* PyThread_get_key_value(int key)
+.. c:function:: void PyThread_delete_key_value(int key)
+.. c:function:: void PyThread_ReInitTLS()
+
diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst
index 74681d2c8518892dd13324281d82e1e82b59b9e7..15006100c7366acc86510e9ef29d186eb0c2b303 100644
--- a/Doc/c-api/intro.rst
+++ b/Doc/c-api/intro.rst
@@ -17,11 +17,11 @@ common use. The second reason is to use Python as a component in a larger
application; this technique is generally referred to as :dfn:`embedding` Python
in an application.
-Writing an extension module is a relatively well-understood process, where a
-"cookbook" approach works well. There are several tools that automate the
-process to some extent. While people have embedded Python in other
-applications since its early existence, the process of embedding Python is less
-straightforward than writing an extension.
+Writing an extension module is a relatively well-understood process, where a
+"cookbook" approach works well. There are several tools that automate the
+process to some extent. While people have embedded Python in other
+applications since its early existence, the process of embedding Python is
+less straightforward than writing an extension.
Many API functions are useful independent of whether you're embedding or
extending Python; moreover, most applications that embed Python will need to
@@ -30,6 +30,16 @@ familiar with writing an extension before attempting to embed Python in a real
application.
+Coding standards
+================
+
+If you're writing C code for inclusion in CPython, you **must** follow the
+guidelines and standards defined in :PEP:`7`. These guidelines apply
+regardless of the version of Python you are contributing to. Following these
+conventions is not necessary for your own third party extension modules,
+unless you eventually expect to contribute them to Python.
+
+
.. _api-includes:
Include Files
@@ -81,6 +91,72 @@ header files do properly declare the entry points to be ``extern "C"``, so there
is no need to do anything special to use the API from C++.
+Useful macros
+=============
+
+Several useful macros are defined in the Python header files. Many are
+defined closer to where they are useful (e.g. :c:macro:`Py_RETURN_NONE`).
+Others of a more general utility are defined here. This is not necessarily a
+complete listing.
+
+.. c:macro:: Py_UNREACHABLE()
+
+ Use this when you have a code path that you do not expect to be reached.
+ For example, in the ``default:`` clause in a ``switch`` statement for which
+ all possible values are covered in ``case`` statements. Use this in places
+ where you might be tempted to put an ``assert(0)`` or ``abort()`` call.
+
+ .. versionadded:: 3.7
+
+.. c:macro:: Py_ABS(x)
+
+ Return the absolute value of ``x``.
+
+ .. versionadded:: 3.3
+
+.. c:macro:: Py_MIN(x, y)
+
+ Return the minimum value between ``x`` and ``y``.
+
+ .. versionadded:: 3.3
+
+.. c:macro:: Py_MAX(x, y)
+
+ Return the maximum value between ``x`` and ``y``.
+
+ .. versionadded:: 3.3
+
+.. c:macro:: Py_STRINGIFY(x)
+
+ Convert ``x`` to a C string. E.g. ``Py_STRINGIFY(123)`` returns
+ ``"123"``.
+
+ .. versionadded:: 3.4
+
+.. c:macro:: Py_MEMBER_SIZE(type, member)
+
+ Return the size of a structure (``type``) ``member`` in bytes.
+
+ .. versionadded:: 3.6
+
+.. c:macro:: Py_CHARMASK(c)
+
+ Argument must be a character or an integer in the range [-128, 127] or [0,
+ 255]. This macro returns ``c`` cast to an ``unsigned char``.
+
+.. c:macro:: Py_GETENV(s)
+
+ Like ``getenv(s)``, but returns *NULL* if :option:`-E` was passed on the
+ command line (i.e. if ``Py_IgnoreEnvironmentFlag`` is set).
+
+.. c:macro:: Py_UNUSED(arg)
+
+ Use this for unused arguments in a function definition to silence compiler
+ warnings, e.g. ``PyObject* func(PyObject *Py_UNUSED(ignored))``.
+
+ .. versionadded:: 3.4
+
+
.. _api-objects:
Objects, Types and Reference Counts
diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst
index c16fcf4439d4dac0c0eae375a7bf7a13d052d283..b8eaadbd702c106e2413d9d6ff282774a169ad3c 100644
--- a/Doc/c-api/mapping.rst
+++ b/Doc/c-api/mapping.rst
@@ -70,17 +70,26 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and
.. c:function:: PyObject* PyMapping_Keys(PyObject *o)
- On success, return a list or tuple of the keys in object *o*. On failure,
- return *NULL*.
+ On success, return a list of the keys in object *o*. On failure, return
+ *NULL*.
+
+ .. versionchanged:: 3.7
+ Previously, the function returned a list or a tuple.
.. c:function:: PyObject* PyMapping_Values(PyObject *o)
- On success, return a list or tuple of the values in object *o*. On failure,
- return *NULL*.
+ On success, return a list of the values in object *o*. On failure, return
+ *NULL*.
+
+ .. versionchanged:: 3.7
+ Previously, the function returned a list or a tuple.
.. c:function:: PyObject* PyMapping_Items(PyObject *o)
- On success, return a list or tuple of the items in object *o*, where each item
- is a tuple containing a key-value pair. On failure, return *NULL*.
+ On success, return a list of the items in object *o*, where each item is a
+ tuple containing a key-value pair. On failure, return *NULL*.
+
+ .. versionchanged:: 3.7
+ Previously, the function returned a list or a tuple.
diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst
index c6d1d02a2fa510e99692b8ba4f2e8cd6f18c8050..17ec621610b5e3aded74939f5fc6ad499830e178 100644
--- a/Doc/c-api/marshal.rst
+++ b/Doc/c-api/marshal.rst
@@ -40,12 +40,6 @@ unmarshalling. Version 2 uses a binary format for floating point numbers.
The following functions allow marshalled values to be read back in.
-XXX What about error detection? It appears that reading past the end of the
-file will always result in a negative numeric value (where that's relevant),
-but it's not clear that negative values won't be handled properly when there's
-no error. What's the right way to tell? Should only non-negative values be
-written using these routines?
-
.. c:function:: long PyMarshal_ReadLongFromFile(FILE *file)
@@ -53,7 +47,8 @@ written using these routines?
for reading. Only a 32-bit value can be read in using this function,
regardless of the native size of :c:type:`long`.
- On error, raise an exception and return ``-1``.
+ On error, sets the appropriate exception (:exc:`EOFError`) and returns
+ ``-1``.
.. c:function:: int PyMarshal_ReadShortFromFile(FILE *file)
@@ -62,7 +57,8 @@ written using these routines?
for reading. Only a 16-bit value can be read in using this function,
regardless of the native size of :c:type:`short`.
- On error, raise an exception and return ``-1``.
+ On error, sets the appropriate exception (:exc:`EOFError`) and returns
+ ``-1``.
.. c:function:: PyObject* PyMarshal_ReadObjectFromFile(FILE *file)
@@ -70,8 +66,8 @@ written using these routines?
Return a Python object from the data stream in a :c:type:`FILE\*` opened for
reading.
- On error, sets the appropriate exception (:exc:`EOFError` or
- :exc:`TypeError`) and returns *NULL*.
+ On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError`
+ or :exc:`TypeError`) and returns *NULL*.
.. c:function:: PyObject* PyMarshal_ReadLastObjectFromFile(FILE *file)
@@ -84,8 +80,8 @@ written using these routines?
file. Only use these variant if you are certain that you won't be reading
anything else from the file.
- On error, sets the appropriate exception (:exc:`EOFError` or
- :exc:`TypeError`) and returns *NULL*.
+ On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError`
+ or :exc:`TypeError`) and returns *NULL*.
.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *data, Py_ssize_t len)
@@ -93,6 +89,6 @@ written using these routines?
Return a Python object from the data stream in a byte buffer
containing *len* bytes pointed to by *data*.
- On error, sets the appropriate exception (:exc:`EOFError` or
- :exc:`TypeError`) and returns *NULL*.
+ On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError`
+ or :exc:`TypeError`) and returns *NULL*.
diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst
index 73bec7c95a1df4030785484ffc17decae93987e6..b79b7e49b67e14cc72d1691a9d62b06bbb4e4673 100644
--- a/Doc/c-api/memory.rst
+++ b/Doc/c-api/memory.rst
@@ -35,7 +35,7 @@ operate within the bounds of the private heap.
It is important to understand that the management of the Python heap is
performed by the interpreter itself and that the user has no control over it,
-even if she regularly manipulates object pointers to memory blocks inside that
+even if they regularly manipulate object pointers to memory blocks inside that
heap. The allocation of heap space for Python objects and other internal
buffers is performed on demand by the Python memory manager through the Python/C
API functions listed in this document.
@@ -100,9 +100,10 @@ The following function sets are wrappers to the system allocator. These
functions are thread-safe, the :term:`GIL ` does not
need to be held.
-The default raw memory block allocator uses the following functions:
-:c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`; call
-``malloc(1)`` (or ``calloc(1, 1)``) when requesting zero bytes.
+The :ref:`default raw memory allocator ` uses
+the following functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc`
+and :c:func:`free`; call ``malloc(1)`` (or ``calloc(1, 1)``) when requesting
+zero bytes.
.. versionadded:: 3.4
@@ -165,7 +166,8 @@ The following function sets, modeled after the ANSI C standard, but specifying
behavior when requesting zero bytes, are available for allocating and releasing
memory from the Python heap.
-By default, these functions use :ref:`pymalloc memory allocator `.
+The :ref:`default memory allocator ` uses the
+:ref:`pymalloc memory allocator `.
.. warning::
@@ -270,7 +272,8 @@ The following function sets, modeled after the ANSI C standard, but specifying
behavior when requesting zero bytes, are available for allocating and releasing
memory from the Python heap.
-By default, these functions use :ref:`pymalloc memory allocator `.
+The :ref:`default object allocator ` uses the
+:ref:`pymalloc memory allocator `.
.. warning::
@@ -326,6 +329,31 @@ By default, these functions use :ref:`pymalloc memory allocator `.
If *p* is *NULL*, no operation is performed.
+.. _default-memory-allocators:
+
+Default Memory Allocators
+=========================
+
+Default memory allocators:
+
+=============================== ==================== ================== ===================== ====================
+Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc
+=============================== ==================== ================== ===================== ====================
+Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc``
+Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug
+Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc``
+Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug
+=============================== ==================== ================== ===================== ====================
+
+Legend:
+
+* Name: value for :envvar:`PYTHONMALLOC` environment variable
+* ``malloc``: system allocators from the standard C library, C functions:
+ :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`
+* ``pymalloc``: :ref:`pymalloc memory allocator `
+* "+ debug": with debug hooks installed by :c:func:`PyMem_SetupDebugHooks`
+
+
Customize Memory Allocators
===========================
@@ -431,7 +459,8 @@ Customize Memory Allocators
displayed if :mod:`tracemalloc` is tracing Python memory allocations and the
memory block was traced.
- These hooks are installed by default if Python is compiled in debug
+ These hooks are :ref:`installed by default ` if
+ Python is compiled in debug
mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install
debug hooks on a Python compiled in release mode.
@@ -450,12 +479,12 @@ The pymalloc allocator
Python has a *pymalloc* allocator optimized for small objects (smaller or equal
to 512 bytes) with a short lifetime. It uses memory mappings called "arenas"
-with a fixed size of 256 KB. It falls back to :c:func:`PyMem_RawMalloc` and
+with a fixed size of 256 KiB. It falls back to :c:func:`PyMem_RawMalloc` and
:c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes.
-*pymalloc* is the default allocator of the :c:data:`PYMEM_DOMAIN_MEM` (ex:
-:c:func:`PyMem_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex:
-:c:func:`PyObject_Malloc`) domains.
+*pymalloc* is the :ref:`default allocator ` of the
+:c:data:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) and
+:c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains.
The arena allocator uses the following functions:
@@ -492,6 +521,28 @@ Customize pymalloc Arena Allocator
Set the arena allocator.
+tracemalloc C API
+=================
+
+.. versionadded:: 3.7
+
+.. c:function: int PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size)
+
+ Track an allocated memory block in the :mod:`tracemalloc` module.
+
+ Return ``0`` on success, return ``-1`` on error (failed to allocate memory to
+ store the trace). Return ``-2`` if tracemalloc is disabled.
+
+ If memory block is already tracked, update the existing trace.
+
+.. c:function: int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr)
+
+ Untrack an allocated memory block in the :mod:`tracemalloc` module.
+ Do nothing if the block was not tracked.
+
+ Return ``-2`` if tracemalloc is disabled, otherwise return ``0``.
+
+
.. _memoryexamples:
Examples
diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst
index 797a67e49c72d8b683f68dc9e1ee13afca906553..017b656854a8cd1e5231d507597ef3edfddeefbe 100644
--- a/Doc/c-api/module.rst
+++ b/Doc/c-api/module.rst
@@ -80,7 +80,7 @@ Module Objects
.. versionadded:: 3.3
-.. c:function:: char* PyModule_GetName(PyObject *module)
+.. c:function:: const char* PyModule_GetName(PyObject *module)
Similar to :c:func:`PyModule_GetNameObject` but return the name encoded to
``'utf-8'``.
@@ -112,7 +112,7 @@ Module Objects
.. versionadded:: 3.2
-.. c:function:: char* PyModule_GetFilename(PyObject *module)
+.. c:function:: const char* PyModule_GetFilename(PyObject *module)
Similar to :c:func:`PyModule_GetFilenameObject` but return the filename
encoded to 'utf-8'.
@@ -146,11 +146,11 @@ or request "multi-phase initialization" by returning the definition struct itsel
Always initialize this member to :const:`PyModuleDef_HEAD_INIT`.
- .. c:member:: char* m_name
+ .. c:member:: const char *m_name
Name for the new module.
- .. c:member:: char* m_doc
+ .. c:member:: const char *m_doc
Docstring for the module; usually a docstring variable created with
:c:func:`PyDoc_STRVAR` is used.
diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst
index 8692a2c14ca20f4fb4365e8f3c04b7161191f894..f0b2005f2d12cc940848f4edf3162c5c88fdb5dd 100644
--- a/Doc/c-api/object.rst
+++ b/Doc/c-api/object.rst
@@ -244,63 +244,82 @@ Object Protocol
and ``0`` otherwise. This function always succeeds.
-.. c:function:: PyObject* PyObject_Call(PyObject *callable_object, PyObject *args, PyObject *kw)
+.. c:function:: PyObject* PyObject_Call(PyObject *callable, PyObject *args, PyObject *kwargs)
- Call a callable Python object *callable_object*, with arguments given by the
- tuple *args*, and named arguments given by the dictionary *kw*. If no named
- arguments are needed, *kw* may be *NULL*. *args* must not be *NULL*, use an
- empty tuple if no arguments are needed. Returns the result of the call on
- success, or *NULL* on failure. This is the equivalent of the Python expression
- ``callable_object(*args, **kw)``.
+ Call a callable Python object *callable*, with arguments given by the
+ tuple *args*, and named arguments given by the dictionary *kwargs*.
+ *args* must not be *NULL*, use an empty tuple if no arguments are needed.
+ If no named arguments are needed, *kwargs* can be *NULL*.
-.. c:function:: PyObject* PyObject_CallObject(PyObject *callable_object, PyObject *args)
+ Returns the result of the call on success, or *NULL* on failure.
- Call a callable Python object *callable_object*, with arguments given by the
- tuple *args*. If no arguments are needed, then *args* may be *NULL*. Returns
- the result of the call on success, or *NULL* on failure. This is the equivalent
- of the Python expression ``callable_object(*args)``.
+ This is the equivalent of the Python expression:
+ ``callable(*args, **kwargs)``.
+
+
+.. c:function:: PyObject* PyObject_CallObject(PyObject *callable, PyObject *args)
+
+ Call a callable Python object *callable*, with arguments given by the
+ tuple *args*. If no arguments are needed, then *args* can be *NULL*.
+
+ Returns the result of the call on success, or *NULL* on failure.
+
+ This is the equivalent of the Python expression: ``callable(*args)``.
.. c:function:: PyObject* PyObject_CallFunction(PyObject *callable, const char *format, ...)
Call a callable Python object *callable*, with a variable number of C arguments.
The C arguments are described using a :c:func:`Py_BuildValue` style format
- string. The format may be *NULL*, indicating that no arguments are provided.
- Returns the result of the call on success, or *NULL* on failure. This is the
- equivalent of the Python expression ``callable(*args)``. Note that if you only
- pass :c:type:`PyObject \*` args, :c:func:`PyObject_CallFunctionObjArgs` is a
- faster alternative.
+ string. The format can be *NULL*, indicating that no arguments are provided.
+
+ Returns the result of the call on success, or *NULL* on failure.
+
+ This is the equivalent of the Python expression: ``callable(*args)``.
+
+ Note that if you only pass :c:type:`PyObject \*` args,
+ :c:func:`PyObject_CallFunctionObjArgs` is a faster alternative.
.. versionchanged:: 3.4
The type of *format* was changed from ``char *``.
-.. c:function:: PyObject* PyObject_CallMethod(PyObject *o, const char *method, const char *format, ...)
+.. c:function:: PyObject* PyObject_CallMethod(PyObject *obj, const char *name, const char *format, ...)
- Call the method named *method* of object *o* with a variable number of C
+ Call the method named *name* of object *obj* with a variable number of C
arguments. The C arguments are described by a :c:func:`Py_BuildValue` format
- string that should produce a tuple. The format may be *NULL*, indicating that
- no arguments are provided. Returns the result of the call on success, or *NULL*
- on failure. This is the equivalent of the Python expression ``o.method(args)``.
+ string that should produce a tuple.
+
+ The format can be *NULL*, indicating that no arguments are provided.
+
+ Returns the result of the call on success, or *NULL* on failure.
+
+ This is the equivalent of the Python expression:
+ ``obj.name(arg1, arg2, ...)``.
+
Note that if you only pass :c:type:`PyObject \*` args,
:c:func:`PyObject_CallMethodObjArgs` is a faster alternative.
.. versionchanged:: 3.4
- The types of *method* and *format* were changed from ``char *``.
+ The types of *name* and *format* were changed from ``char *``.
.. c:function:: PyObject* PyObject_CallFunctionObjArgs(PyObject *callable, ..., NULL)
Call a callable Python object *callable*, with a variable number of
:c:type:`PyObject\*` arguments. The arguments are provided as a variable number
- of parameters followed by *NULL*. Returns the result of the call on success, or
- *NULL* on failure.
+ of parameters followed by *NULL*.
+
+ Returns the result of the call on success, or *NULL* on failure.
+
+ This is the equivalent of the Python expression:
+ ``callable(arg1, arg2, ...)``.
-.. c:function:: PyObject* PyObject_CallMethodObjArgs(PyObject *o, PyObject *name, ..., NULL)
+.. c:function:: PyObject* PyObject_CallMethodObjArgs(PyObject *obj, PyObject *name, ..., NULL)
- Calls a method of the object *o*, where the name of the method is given as a
+ Calls a method of the Python object *obj*, where the name of the method is given as a
Python string object in *name*. It is called with a variable number of
:c:type:`PyObject\*` arguments. The arguments are provided as a variable number
of parameters followed by *NULL*. Returns the result of the call on success, or
diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst
index 8b695e065aeffd9532176c5d22f86e8c7fa392e3..8ad9a29b256ebcedb4569f7299d1118cdb094365 100644
--- a/Doc/c-api/slice.rst
+++ b/Doc/c-api/slice.rst
@@ -53,10 +53,63 @@ Slice Objects
Returns ``0`` on success and ``-1`` on error with exception set.
+ .. note::
+ This function is considered not safe for resizable sequences.
+ Its invocation should be replaced by a combination of
+ :c:func:`PySlice_Unpack` and :c:func:`PySlice_AdjustIndices` where ::
+
+ if (PySlice_GetIndicesEx(slice, length, &start, &stop, &step, &slicelength) < 0) {
+ // return error
+ }
+
+ is replaced by ::
+
+ if (PySlice_Unpack(slice, &start, &stop, &step) < 0) {
+ // return error
+ }
+ slicelength = PySlice_AdjustIndices(length, &start, &stop, step);
+
.. versionchanged:: 3.2
The parameter type for the *slice* parameter was ``PySliceObject*``
before.
+ .. versionchanged:: 3.6.1
+ If ``Py_LIMITED_API`` is not set or set to the value between ``0x03050400``
+ and ``0x03060000`` (not including) or ``0x03060100`` or higher
+ :c:func:`!PySlice_GetIndicesEx` is implemented as a macro using
+ :c:func:`!PySlice_Unpack` and :c:func:`!PySlice_AdjustIndices`.
+ Arguments *start*, *stop* and *step* are evaluated more than once.
+
+ .. deprecated:: 3.6.1
+ If ``Py_LIMITED_API`` is set to the value less than ``0x03050400`` or
+ between ``0x03060000`` and ``0x03060100`` (not including)
+ :c:func:`!PySlice_GetIndicesEx` is a deprecated function.
+
+
+.. c:function:: int PySlice_Unpack(PyObject *slice, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step)
+
+ Extract the start, stop and step data members from a slice object as
+ C integers. Silently reduce values larger than ``PY_SSIZE_T_MAX`` to
+ ``PY_SSIZE_T_MAX``, silently boost the start and stop values less than
+ ``PY_SSIZE_T_MIN`` to ``PY_SSIZE_T_MIN``, and silently boost the step
+ values less than ``-PY_SSIZE_T_MAX`` to ``-PY_SSIZE_T_MAX``.
+
+ Return ``-1`` on error, ``0`` on success.
+
+ .. versionadded:: 3.6.1
+
+
+.. c:function:: Py_ssize_t PySlice_AdjustIndices(Py_ssize_t length, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t step)
+
+ Adjust start/end slice indices assuming a sequence of the specified length.
+ Out of bounds indices are clipped in a manner consistent with the handling
+ of normal slices.
+
+ Return the length of the slice. Always successful. Doesn't call Python
+ code.
+
+ .. versionadded:: 3.6.1
+
Ellipsis Object
---------------
diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst
index 675f6f26921c55d438deaa8ba968485387c313ff..da45da1d3c70dbc57de63e45a9337e2c573faa45 100644
--- a/Doc/c-api/structures.rst
+++ b/Doc/c-api/structures.rst
@@ -125,20 +125,20 @@ the definition of all other Python objects.
Structure used to describe a method of an extension type. This structure has
four fields:
- +------------------+-------------+-------------------------------+
- | Field | C Type | Meaning |
- +==================+=============+===============================+
- | :attr:`ml_name` | char \* | name of the method |
- +------------------+-------------+-------------------------------+
- | :attr:`ml_meth` | PyCFunction | pointer to the C |
- | | | implementation |
- +------------------+-------------+-------------------------------+
- | :attr:`ml_flags` | int | flag bits indicating how the |
- | | | call should be constructed |
- +------------------+-------------+-------------------------------+
- | :attr:`ml_doc` | char \* | points to the contents of the |
- | | | docstring |
- +------------------+-------------+-------------------------------+
+ +------------------+---------------+-------------------------------+
+ | Field | C Type | Meaning |
+ +==================+===============+===============================+
+ | :attr:`ml_name` | const char \* | name of the method |
+ +------------------+---------------+-------------------------------+
+ | :attr:`ml_meth` | PyCFunction | pointer to the C |
+ | | | implementation |
+ +------------------+---------------+-------------------------------+
+ | :attr:`ml_flags` | int | flag bits indicating how the |
+ | | | call should be constructed |
+ +------------------+---------------+-------------------------------+
+ | :attr:`ml_doc` | const char \* | points to the contents of the |
+ | | | docstring |
+ +------------------+---------------+-------------------------------+
The :attr:`ml_meth` is a C function pointer. The functions may be of different
types, but they always return :c:type:`PyObject\*`. If the function is not of
@@ -236,25 +236,25 @@ definition with the same method name.
Structure which describes an attribute of a type which corresponds to a C
struct member. Its fields are:
- +------------------+-------------+-------------------------------+
- | Field | C Type | Meaning |
- +==================+=============+===============================+
- | :attr:`name` | char \* | name of the member |
- +------------------+-------------+-------------------------------+
- | :attr:`!type` | int | the type of the member in the |
- | | | C struct |
- +------------------+-------------+-------------------------------+
- | :attr:`offset` | Py_ssize_t | the offset in bytes that the |
- | | | member is located on the |
- | | | type's object struct |
- +------------------+-------------+-------------------------------+
- | :attr:`flags` | int | flag bits indicating if the |
- | | | field should be read-only or |
- | | | writable |
- +------------------+-------------+-------------------------------+
- | :attr:`doc` | char \* | points to the contents of the |
- | | | docstring |
- +------------------+-------------+-------------------------------+
+ +------------------+---------------+-------------------------------+
+ | Field | C Type | Meaning |
+ +==================+===============+===============================+
+ | :attr:`name` | const char \* | name of the member |
+ +------------------+---------------+-------------------------------+
+ | :attr:`!type` | int | the type of the member in the |
+ | | | C struct |
+ +------------------+---------------+-------------------------------+
+ | :attr:`offset` | Py_ssize_t | the offset in bytes that the |
+ | | | member is located on the |
+ | | | type's object struct |
+ +------------------+---------------+-------------------------------+
+ | :attr:`flags` | int | flag bits indicating if the |
+ | | | field should be read-only or |
+ | | | writable |
+ +------------------+---------------+-------------------------------+
+ | :attr:`doc` | const char \* | points to the contents of the |
+ | | | docstring |
+ +------------------+---------------+-------------------------------+
:attr:`!type` can be one of many ``T_`` macros corresponding to various C
types. When the member is accessed in Python, it will be converted to the
@@ -268,7 +268,7 @@ definition with the same method name.
T_LONG long
T_FLOAT float
T_DOUBLE double
- T_STRING char \*
+ T_STRING const char \*
T_OBJECT PyObject \*
T_OBJECT_EX PyObject \*
T_CHAR char
@@ -292,7 +292,8 @@ definition with the same method name.
:attr:`flags` can be ``0`` for write and read access or :c:macro:`READONLY` for
read-only access. Using :c:macro:`T_STRING` for :attr:`type` implies
- :c:macro:`READONLY`. Only :c:macro:`T_OBJECT` and :c:macro:`T_OBJECT_EX`
+ :c:macro:`READONLY`. :c:macro:`T_STRING` data is interpreted as UTF-8.
+ Only :c:macro:`T_OBJECT` and :c:macro:`T_OBJECT_EX`
members can be deleted. (They are set to *NULL*).
@@ -304,7 +305,7 @@ definition with the same method name.
+-------------+------------------+-----------------------------------+
| Field | C Type | Meaning |
+=============+==================+===================================+
- | name | char \* | attribute name |
+ | name | const char \* | attribute name |
+-------------+------------------+-----------------------------------+
| get | getter | C Function to get the attribute |
+-------------+------------------+-----------------------------------+
@@ -312,7 +313,7 @@ definition with the same method name.
| | | delete the attribute, if omitted |
| | | the attribute is readonly |
+-------------+------------------+-----------------------------------+
- | doc | char \* | optional docstring |
+ | doc | const char \* | optional docstring |
+-------------+------------------+-----------------------------------+
| closure | void \* | optional function pointer, |
| | | providing additional data for |
diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst
index 48e2b2bb5e91a6f6cbfa454471fee4c1590ddfa9..994509aa50f2ad38bb2b327f1de95425d1c81921 100644
--- a/Doc/c-api/sys.rst
+++ b/Doc/c-api/sys.rst
@@ -26,6 +26,43 @@ Operating System Utilities
one of the strings ``''`` or ``'???'``.
+.. c:function:: void PyOS_BeforeFork()
+
+ Function to prepare some internal state before a process fork. This
+ should be called before calling :c:func:`fork` or any similar function
+ that clones the current process.
+ Only available on systems where :c:func:`fork` is defined.
+
+ .. versionadded:: 3.7
+
+
+.. c:function:: void PyOS_AfterFork_Parent()
+
+ Function to update some internal state after a process fork. This
+ should be called from the parent process after calling :c:func:`fork`
+ or any similar function that clones the current process, regardless
+ of whether process cloning was successful.
+ Only available on systems where :c:func:`fork` is defined.
+
+ .. versionadded:: 3.7
+
+
+.. c:function:: void PyOS_AfterFork_Child()
+
+ Function to update internal interpreter state after a process fork.
+ This must be called from the child process after calling :c:func:`fork`,
+ or any similar function that clones the current process, if there is
+ any chance the process will call back into the Python interpreter.
+ Only available on systems where :c:func:`fork` is defined.
+
+ .. versionadded:: 3.7
+
+ .. seealso::
+ :func:`os.register_at_fork` allows registering custom Python functions
+ to be called by :c:func:`PyOS_BeforeFork()`,
+ :c:func:`PyOS_AfterFork_Parent` and :c:func:`PyOS_AfterFork_Child`.
+
+
.. c:function:: void PyOS_AfterFork()
Function to update some internal state after a process fork; this should be
@@ -33,6 +70,9 @@ Operating System Utilities
If a new executable is loaded into the new process, this function does not need
to be called.
+ .. deprecated:: 3.7
+ This function is superseded by :c:func:`PyOS_AfterFork_Child()`.
+
.. c:function:: int PyOS_CheckStack()
@@ -69,15 +109,16 @@ Operating System Utilities
Encoding, highest priority to lowest priority:
* ``UTF-8`` on macOS and Android;
+ * ``UTF-8`` if the Python UTF-8 mode is enabled;
* ``ASCII`` if the ``LC_CTYPE`` locale is ``"C"``,
``nl_langinfo(CODESET)`` returns the ``ASCII`` encoding (or an alias),
- and :c:func:`mbstowcs` and :c:func:`wcstombs` functions use the
+ and :c:func:`mbstowcs` and :c:func:`wcstombs` functions uses the
``ISO-8859-1`` encoding.
- * the current locale encoding (``LC_CTYPE`` locale).
+ * the current locale encoding.
Return a pointer to a newly allocated wide character string, use
:c:func:`PyMem_RawFree` to free the memory. If size is not ``NULL``, write
- the number of wide characters excluding the null character into ``*size``.
+ the number of wide characters excluding the null character into ``*size``
Return ``NULL`` on decoding error or memory allocation error. If *size* is
not ``NULL``, ``*size`` is set to ``(size_t)-1`` on memory error or set to
@@ -96,6 +137,9 @@ Operating System Utilities
.. versionadded:: 3.5
+ .. versionchanged:: 3.7
+ The function now uses the UTF-8 encoding in the UTF-8 mode.
+
.. c:function:: char* Py_EncodeLocale(const wchar_t *text, size_t *error_pos)
@@ -106,22 +150,28 @@ Operating System Utilities
Encoding, highest priority to lowest priority:
* ``UTF-8`` on macOS and Android;
+ * ``UTF-8`` if the Python UTF-8 mode is enabled;
* ``ASCII`` if the ``LC_CTYPE`` locale is ``"C"``,
``nl_langinfo(CODESET)`` returns the ``ASCII`` encoding (or an alias),
and :c:func:`mbstowcs` and :c:func:`wcstombs` functions uses the
``ISO-8859-1`` encoding.
* the current locale encoding.
+ The function uses the UTF-8 encoding in the Python UTF-8 mode.
+
Return a pointer to a newly allocated byte string, use :c:func:`PyMem_Free`
to free the memory. Return ``NULL`` on encoding error or memory allocation
error
- If error_pos is not ``NULL``, ``*error_pos`` is set to the index of the
- invalid character on encoding error, or set to ``(size_t)-1`` otherwise.
+ If error_pos is not ``NULL``, ``*error_pos`` is set to ``(size_t)-1`` on
+ success, or set to the index of the invalid character on encoding error.
Use the :c:func:`Py_DecodeLocale` function to decode the bytes string back
to a wide character string.
+ .. versionchanged:: 3.7
+ The function now uses the UTF-8 encoding in the UTF-8 mode.
+
.. seealso::
The :c:func:`PyUnicode_EncodeFSDefault` and
@@ -129,6 +179,9 @@ Operating System Utilities
.. versionadded:: 3.5
+ .. versionchanged:: 3.7
+ The function now supports the UTF-8 mode.
+
.. _systemfunctions:
@@ -152,17 +205,25 @@ accessible to C code. They all work with the current interpreter thread's
.. c:function:: void PySys_ResetWarnOptions()
- Reset :data:`sys.warnoptions` to an empty list.
+ Reset :data:`sys.warnoptions` to an empty list. This function may be
+ called prior to :c:func:`Py_Initialize`.
-.. c:function:: void PySys_AddWarnOption(wchar_t *s)
+.. c:function:: void PySys_AddWarnOption(const wchar_t *s)
- Append *s* to :data:`sys.warnoptions`.
+ Append *s* to :data:`sys.warnoptions`. This function must be called prior
+ to :c:func:`Py_Initialize` in order to affect the warnings filter list.
.. c:function:: void PySys_AddWarnOptionUnicode(PyObject *unicode)
Append *unicode* to :data:`sys.warnoptions`.
-.. c:function:: void PySys_SetPath(wchar_t *path)
+ Note: this function is not currently usable from outside the CPython
+ implementation, as it must be called prior to the implicit import of
+ :mod:`warnings` in :c:func:`Py_Initialize` to be effective, but can't be
+ called until enough of the runtime has been initialized to permit the
+ creation of Unicode objects.
+
+.. c:function:: void PySys_SetPath(const wchar_t *path)
Set :data:`sys.path` to a list object of paths found in *path* which should
be a list of paths separated with the platform's search path delimiter
@@ -207,7 +268,8 @@ accessible to C code. They all work with the current interpreter thread's
.. c:function:: void PySys_AddXOption(const wchar_t *s)
Parse *s* as a set of :option:`-X` options and add them to the current
- options mapping as returned by :c:func:`PySys_GetXOptions`.
+ options mapping as returned by :c:func:`PySys_GetXOptions`. This function
+ may be called prior to :c:func:`Py_Initialize`.
.. versionadded:: 3.2
diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst
index 3922d50f80a2d217bc00f066e753bf413ef341d9..a66832cfa4349280dd8a633978858d5b90243906 100644
--- a/Doc/c-api/tuple.rst
+++ b/Doc/c-api/tuple.rst
@@ -144,9 +144,9 @@ type.
+-------------------+------------------------------+------------------------------------+
| Field | C Type | Meaning |
+===================+==============================+====================================+
- | ``name`` | ``char *`` | name of the struct sequence type |
+ | ``name`` | ``const char *`` | name of the struct sequence type |
+-------------------+------------------------------+------------------------------------+
- | ``doc`` | ``char *`` | pointer to docstring for the type |
+ | ``doc`` | ``const char *`` | pointer to docstring for the type |
| | | or NULL to omit |
+-------------------+------------------------------+------------------------------------+
| ``fields`` | ``PyStructSequence_Field *`` | pointer to *NULL*-terminated array |
@@ -164,16 +164,16 @@ type.
:attr:`fields` array of the :c:type:`PyStructSequence_Desc` determines which
field of the struct sequence is described.
- +-----------+---------------+--------------------------------------+
- | Field | C Type | Meaning |
- +===========+===============+======================================+
- | ``name`` | ``char *`` | name for the field or *NULL* to end |
- | | | the list of named fields, set to |
- | | | PyStructSequence_UnnamedField to |
- | | | leave unnamed |
- +-----------+---------------+--------------------------------------+
- | ``doc`` | ``char *`` | field docstring or *NULL* to omit |
- +-----------+---------------+--------------------------------------+
+ +-----------+------------------+--------------------------------------+
+ | Field | C Type | Meaning |
+ +===========+==================+======================================+
+ | ``name`` | ``const char *`` | name for the field or *NULL* to end |
+ | | | the list of named fields, set to |
+ | | | PyStructSequence_UnnamedField to |
+ | | | leave unnamed |
+ +-----------+------------------+--------------------------------------+
+ | ``doc`` | ``const char *`` | field docstring or *NULL* to omit |
+ +-----------+------------------+--------------------------------------+
.. c:var:: char* PyStructSequence_UnnamedField
diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst
index 60c5e73960b3a1f9133f2d937ee7711eac48e725..4dfd53fb9f076fc6a3906b4e840c4c117deb96ab 100644
--- a/Doc/c-api/type.rst
+++ b/Doc/c-api/type.rst
@@ -35,7 +35,7 @@ Type Objects
Clear the internal lookup cache. Return the current version tag.
-.. c:function:: long PyType_GetFlags(PyTypeObject* type)
+.. c:function:: unsigned long PyType_GetFlags(PyTypeObject* type)
Return the :c:member:`~PyTypeObject.tp_flags` member of *type*. This function is primarily
meant for use with `Py_LIMITED_API`; the individual flag bits are
@@ -44,6 +44,9 @@ Type Objects
.. versionadded:: 3.2
+ .. versionchanged:: 3.4
+ The return type is now ``unsigned long`` rather than ``long``.
+
.. c:function:: void PyType_Modified(PyTypeObject *type)
diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst
index 76515fd2c430e358fee0a0e5209e241f14a4653b..6cbcc273c1f153b5260d75e4d10654a4ece5dffe 100644
--- a/Doc/c-api/typeobj.rst
+++ b/Doc/c-api/typeobj.rst
@@ -623,6 +623,22 @@ type objects) *must* have the :attr:`ob_size` field.
| :const:`Py_GE` | ``>=`` |
+----------------+------------+
+ The following macro is defined to ease writing rich comparison functions:
+
+ .. c:function:: PyObject *Py_RETURN_RICHCOMPARE(VAL_A, VAL_B, int op)
+
+ Return ``Py_True`` or ``Py_False`` from the function, depending on the
+ result of a comparison.
+ VAL_A and VAL_B must be orderable by C comparison operators (for example,
+ they may be C ints or floats). The third argument specifies the requested
+ operation, as for :c:func:`PyObject_RichCompare`.
+
+ The return value's reference count is properly incremented.
+
+ On error, sets an exception and returns NULL from the function.
+
+ .. versionadded:: 3.7
+
.. c:member:: Py_ssize_t PyTypeObject.tp_weaklistoffset
diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst
index b9acaec949b252e78b94759e440009265b2eb79a..92e22b16a4ef2992db976ae06a006574b35c01d3 100644
--- a/Doc/c-api/unicode.rst
+++ b/Doc/c-api/unicode.rst
@@ -451,49 +451,49 @@ APIs:
| :attr:`%c` | int | A single character, |
| | | represented as a C int. |
+-------------------+---------------------+--------------------------------+
- | :attr:`%d` | int | Exactly equivalent to |
- | | | ``printf("%d")``. |
+ | :attr:`%d` | int | Equivalent to |
+ | | | ``printf("%d")``. [1]_ |
+-------------------+---------------------+--------------------------------+
- | :attr:`%u` | unsigned int | Exactly equivalent to |
- | | | ``printf("%u")``. |
+ | :attr:`%u` | unsigned int | Equivalent to |
+ | | | ``printf("%u")``. [1]_ |
+-------------------+---------------------+--------------------------------+
- | :attr:`%ld` | long | Exactly equivalent to |
- | | | ``printf("%ld")``. |
+ | :attr:`%ld` | long | Equivalent to |
+ | | | ``printf("%ld")``. [1]_ |
+-------------------+---------------------+--------------------------------+
- | :attr:`%li` | long | Exactly equivalent to |
- | | | ``printf("%li")``. |
+ | :attr:`%li` | long | Equivalent to |
+ | | | ``printf("%li")``. [1]_ |
+-------------------+---------------------+--------------------------------+
- | :attr:`%lu` | unsigned long | Exactly equivalent to |
- | | | ``printf("%lu")``. |
+ | :attr:`%lu` | unsigned long | Equivalent to |
+ | | | ``printf("%lu")``. [1]_ |
+-------------------+---------------------+--------------------------------+
- | :attr:`%lld` | long long | Exactly equivalent to |
- | | | ``printf("%lld")``. |
+ | :attr:`%lld` | long long | Equivalent to |
+ | | | ``printf("%lld")``. [1]_ |
+-------------------+---------------------+--------------------------------+
- | :attr:`%lli` | long long | Exactly equivalent to |
- | | | ``printf("%lli")``. |
+ | :attr:`%lli` | long long | Equivalent to |
+ | | | ``printf("%lli")``. [1]_ |
+-------------------+---------------------+--------------------------------+
- | :attr:`%llu` | unsigned long long | Exactly equivalent to |
- | | | ``printf("%llu")``. |
+ | :attr:`%llu` | unsigned long long | Equivalent to |
+ | | | ``printf("%llu")``. [1]_ |
+-------------------+---------------------+--------------------------------+
- | :attr:`%zd` | Py_ssize_t | Exactly equivalent to |
- | | | ``printf("%zd")``. |
+ | :attr:`%zd` | Py_ssize_t | Equivalent to |
+ | | | ``printf("%zd")``. [1]_ |
+-------------------+---------------------+--------------------------------+
- | :attr:`%zi` | Py_ssize_t | Exactly equivalent to |
- | | | ``printf("%zi")``. |
+ | :attr:`%zi` | Py_ssize_t | Equivalent to |
+ | | | ``printf("%zi")``. [1]_ |
+-------------------+---------------------+--------------------------------+
- | :attr:`%zu` | size_t | Exactly equivalent to |
- | | | ``printf("%zu")``. |
+ | :attr:`%zu` | size_t | Equivalent to |
+ | | | ``printf("%zu")``. [1]_ |
+-------------------+---------------------+--------------------------------+
- | :attr:`%i` | int | Exactly equivalent to |
- | | | ``printf("%i")``. |
+ | :attr:`%i` | int | Equivalent to |
+ | | | ``printf("%i")``. [1]_ |
+-------------------+---------------------+--------------------------------+
- | :attr:`%x` | int | Exactly equivalent to |
- | | | ``printf("%x")``. |
+ | :attr:`%x` | int | Equivalent to |
+ | | | ``printf("%x")``. [1]_ |
+-------------------+---------------------+--------------------------------+
- | :attr:`%s` | char\* | A null-terminated C character |
+ | :attr:`%s` | const char\* | A null-terminated C character |
| | | array. |
+-------------------+---------------------+--------------------------------+
- | :attr:`%p` | void\* | The hex representation of a C |
+ | :attr:`%p` | const void\* | The hex representation of a C |
| | | pointer. Mostly equivalent to |
| | | ``printf("%p")`` except that |
| | | it is guaranteed to start with |
@@ -506,8 +506,8 @@ APIs:
+-------------------+---------------------+--------------------------------+
| :attr:`%U` | PyObject\* | A unicode object. |
+-------------------+---------------------+--------------------------------+
- | :attr:`%V` | PyObject\*, char \* | A unicode object (which may be |
- | | | *NULL*) and a null-terminated |
+ | :attr:`%V` | PyObject\*, | A unicode object (which may be |
+ | | const char\* | *NULL*) and a null-terminated |
| | | C character array as a second |
| | | parameter (which will be used, |
| | | if the first parameter is |
@@ -530,6 +530,9 @@ APIs:
characters for ``"%A"``, ``"%U"``, ``"%S"``, ``"%R"`` and ``"%V"``
(if the ``PyObject*`` argument is not NULL).
+ .. [1] For integer specifiers (d, u, ld, li, lu, lld, lli, llu, zd, zi,
+ zu, i, x): the 0-conversion flag has effect even when a precision is given.
+
.. versionchanged:: 3.2
Support for ``"%lld"`` and ``"%llu"`` added.
@@ -757,7 +760,8 @@ system.
Py_ssize_t len, \
const char *errors)
- Decode a string from the current locale encoding. The supported
+ Decode a string from UTF-8 on Android, or from the current locale encoding
+ on other platforms. The supported
error handlers are ``"strict"`` and ``"surrogateescape"``
(:pep:`383`). The decoder uses ``"strict"`` error handler if
*errors* is ``NULL``. *str* must end with a null character but
@@ -767,15 +771,17 @@ system.
:c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at
Python startup).
+ This function ignores the Python UTF-8 mode.
+
.. seealso::
The :c:func:`Py_DecodeLocale` function.
.. versionadded:: 3.3
- .. versionchanged:: 3.6.5
+ .. versionchanged:: 3.7
The function now also uses the current locale encoding for the
- ``surrogateescape`` error handler. Previously, :c:func:`Py_DecodeLocale`
+ ``surrogateescape`` error handler, except on Android. Previously, :c:func:`Py_DecodeLocale`
was used for the ``surrogateescape``, and the current locale encoding was
used for ``strict``.
@@ -790,7 +796,8 @@ system.
.. c:function:: PyObject* PyUnicode_EncodeLocale(PyObject *unicode, const char *errors)
- Encode a Unicode object to the current locale encoding. The
+ Encode a Unicode object to UTF-8 on Android, or to the current locale
+ encoding on other platforms. The
supported error handlers are ``"strict"`` and ``"surrogateescape"``
(:pep:`383`). The encoder uses ``"strict"`` error handler if
*errors* is ``NULL``. Return a :class:`bytes` object. *unicode* cannot
@@ -800,15 +807,18 @@ system.
:c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at
Python startup).
+ This function ignores the Python UTF-8 mode.
+
.. seealso::
The :c:func:`Py_EncodeLocale` function.
.. versionadded:: 3.3
- .. versionchanged:: 3.6.5
+ .. versionchanged:: 3.7
The function now also uses the current locale encoding for the
- ``surrogateescape`` error handler. Previously, :c:func:`Py_EncodeLocale`
+ ``surrogateescape`` error handler, except on Android. Previously,
+ :c:func:`Py_EncodeLocale`
was used for the ``surrogateescape``, and the current locale encoding was
used for ``strict``.
@@ -943,16 +953,22 @@ wchar_t Support
Convert the Unicode object to a wide character string. The output string
always ends with a null character. If *size* is not *NULL*, write the number
of wide characters (excluding the trailing null termination character) into
- *\*size*.
+ *\*size*. Note that the resulting :c:type:`wchar_t` string might contain
+ null characters, which would cause the string to be truncated when used with
+ most C functions. If *size* is *NULL* and the :c:type:`wchar_t*` string
+ contains null characters a :exc:`ValueError` is raised.
Returns a buffer allocated by :c:func:`PyMem_Alloc` (use
- :c:func:`PyMem_Free` to free it) on success. On error, returns *NULL*,
- *\*size* is undefined and raises a :exc:`MemoryError`. Note that the
- resulting :c:type:`wchar_t` string might contain null characters, which
- would cause the string to be truncated when used with most C functions.
+ :c:func:`PyMem_Free` to free it) on success. On error, returns *NULL*
+ and *\*size* is undefined. Raises a :exc:`MemoryError` if memory allocation
+ is failed.
.. versionadded:: 3.2
+ .. versionchanged:: 3.7
+ Raises a :exc:`ValueError` if *size* is *NULL* and the :c:type:`wchar_t*`
+ string contains null characters.
+
.. _builtincodecs:
@@ -1050,7 +1066,7 @@ These are the UTF-8 codec APIs:
raised by the codec.
-.. c:function:: char* PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *size)
+.. c:function:: const char* PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *size)
Return a pointer to the UTF-8 encoding of the Unicode object, and
store the size of the encoded representation (in bytes) in *size*. The
@@ -1067,13 +1083,19 @@ These are the UTF-8 codec APIs:
.. versionadded:: 3.3
+ .. versionchanged:: 3.7
+ The return type is now ``const char *`` rather of ``char *``.
-.. c:function:: char* PyUnicode_AsUTF8(PyObject *unicode)
+
+.. c:function:: const char* PyUnicode_AsUTF8(PyObject *unicode)
As :c:func:`PyUnicode_AsUTF8AndSize`, but does not store the size.
.. versionadded:: 3.3
+ .. versionchanged:: 3.7
+ The return type is now ``const char *`` rather of ``char *``.
+
.. c:function:: PyObject* PyUnicode_EncodeUTF8(const Py_UNICODE *s, Py_ssize_t size, const char *errors)
@@ -1618,6 +1640,9 @@ They all return *NULL* or ``-1`` if an exception occurs.
.. versionadded:: 3.3
+ .. versionchanged:: 3.7
+ *start* and *end* are now adjusted to behave like ``str[start:end]``.
+
.. c:function:: Py_ssize_t PyUnicode_Count(PyObject *str, PyObject *substr, \
Py_ssize_t start, Py_ssize_t end)
diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst
index 3897fdd828216dd33826faf0472e0eb5dc13df66..c891f6320f944f39f4d67e0a104ea5fef648c804 100644
--- a/Doc/c-api/veryhigh.rst
+++ b/Doc/c-api/veryhigh.rst
@@ -301,12 +301,12 @@ the same library that the Python runtime is using.
set to *NULL*.
-.. c:function:: PyObject* PyEval_EvalCodeEx(PyObject *co, PyObject *globals, PyObject *locals, PyObject **args, int argcount, PyObject **kws, int kwcount, PyObject **defs, int defcount, PyObject *kwdefs, PyObject *closure)
+.. c:function:: PyObject* PyEval_EvalCodeEx(PyObject *co, PyObject *globals, PyObject *locals, PyObject *const *args, int argcount, PyObject *const *kws, int kwcount, PyObject *const *defs, int defcount, PyObject *kwdefs, PyObject *closure)
Evaluate a precompiled code object, given a particular environment for its
evaluation. This environment consists of a dictionary of global variables,
a mapping object of local variables, arrays of arguments, keywords and
- defaults, a dictionary of default values for :ref:`keyword-only\
+ defaults, a dictionary of default values for :ref:`keyword-only
` arguments and a closure tuple of cells.
diff --git a/Doc/conf.py b/Doc/conf.py
index 43826ec01f46634d8c82070e68a4427c0f7dcca6..eab3c39cfb7ccc0bc87248f76ac0d5b470bf2be6 100644
--- a/Doc/conf.py
+++ b/Doc/conf.py
@@ -8,12 +8,13 @@
import sys, os, time
sys.path.append(os.path.abspath('tools/extensions'))
+sys.path.append(os.path.abspath('includes'))
# General configuration
# ---------------------
extensions = ['sphinx.ext.coverage', 'sphinx.ext.doctest',
- 'pyspecific', 'c_annotations']
+ 'pyspecific', 'c_annotations', 'escape4chm']
# General substitutions.
project = 'Python'
@@ -33,13 +34,18 @@ today_fmt = '%B %d, %Y'
# By default, highlight as Python 3.
highlight_language = 'python3'
-# Require Sphinx 1.2 for build.
-needs_sphinx = '1.2'
+# Require Sphinx 1.6.6 for build.
+needs_sphinx = "1.6.6"
# Ignore any .rst files in the venv/ directory.
venvdir = os.getenv('VENVDIR', 'venv')
exclude_patterns = [venvdir+'/*', 'README.rst']
+# Disable Docutils smartquotes for several translations
+smartquotes_excludes = {
+ 'languages': ['ja', 'fr', 'zh_TW', 'zh_CN'], 'builders': ['man', 'text'],
+}
+
# Options for HTML output
# -----------------------
diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat
index 8b469f4aac9a489ff12a1fe9a9870f456057d723..62cc93832783b7ef770f0f3c9f39c64d71063ecf 100644
--- a/Doc/data/refcounts.dat
+++ b/Doc/data/refcounts.dat
@@ -152,6 +152,47 @@ PyComplex_ImagAsDouble:PyObject*:op:0:
PyComplex_RealAsDouble:double:::
PyComplex_RealAsDouble:PyObject*:op:0:
+PyContext_CheckExact:int:::
+PyContext_CheckExact:PyObject*:o:0:
+
+PyContext_ClearFreeList:int:::
+
+PyContext_Copy:PyObject*::+1:
+PyContext_Copy:PyObject*:ctx:0:
+
+PyContext_CopyCurrent:PyObject*::+1:
+
+PyContext_Enter:int:::
+PyContext_Enter:PyObject*:ctx:+1:
+
+PyContext_Exit:int:::
+PyContext_Exit:PyObject*:ctx:-1:
+
+PyContext_New:PyObject*::+1:
+
+PyContextToken_CheckExact:int:::
+PyContextToken_CheckExact:PyObject*:o:0:
+
+PyContextVar_CheckExact:int:::
+PyContextVar_CheckExact:PyObject*:o:0:
+
+PyContextVar_Get:int:::
+PyContextVar_Get:PyObject*:var:0:
+PyContextVar_Get:PyObject*:default_value:0:
+PyContextVar_Get:PyObject**:value:+1:???
+
+PyContextVar_New:PyObject*::+1:
+PyContextVar_New:const char*:name::
+PyContextVar_New:PyObject*:def:+1:
+
+PyContextVar_Set:PyObject*::+1:
+PyContextVar_Set:PyObject*:var:0:
+PyContextVar_Set:PyObject*:value:+1:
+
+PyContextVar_Reset:int:::
+PyContextVar_Reset:PyObject*:var:0:
+PyContextVar_Reset:PyObject*:token:-1:
+
PyDate_FromDate:PyObject*::+1:
PyDate_FromDate:int:year::
PyDate_FromDate:int:month::
@@ -177,6 +218,14 @@ PyDelta_FromDSU:int:days::
PyDelta_FromDSU:int:seconds::
PyDelta_FromDSU:int:useconds::
+PyTimeZone_FromOffset:PyObject*::+1:
+PyTimeZone_FromOffset:PyDateTime_DeltaType*:offset:+1:Reference count not increased if offset is +00:00
+
+PyTimeZone_FromOffsetAndName:PyObject*::+1:
+PyTimeZone_FromOffsetAndName:PyDateTime_DeltaType*:offset:+1:Reference count not increased if offset is +00:00 and name == NULL
+PyTimeZone_FromOffsetAndName:PyUnicode*:name:+1:
+
+
PyDescr_NewClassMethod:PyObject*::+1:
PyDescr_NewClassMethod:PyTypeObject*:type::
PyDescr_NewClassMethod:PyMethodDef*:method::
@@ -216,6 +265,10 @@ PyDict_GetItem:PyObject*::0:0
PyDict_GetItem:PyObject*:p:0:
PyDict_GetItem:PyObject*:key:0:
+PyDict_GetItemWithError:PyObject*::0:0
+PyDict_GetItemWithError:PyObject*:p:0:
+PyDict_GetItemWithError:PyObject*:key:0:
+
PyDict_GetItemString:PyObject*::0:
PyDict_GetItemString:PyObject*:p:0:
PyDict_GetItemString:const char*:key::
@@ -529,6 +582,9 @@ PyImport_ExecCodeModuleEx:const char*:pathname::
PyImport_GetMagicNumber:long:::
+PyImport_GetModule:PyObject*::+1:
+PyImport_GetModule:PyObject*:name:0:
+
PyImport_GetModuleDict:PyObject*::0:
PyImport_Import:PyObject*::+1:
@@ -593,6 +649,9 @@ PyInterpreterState_Clear:PyInterpreterState*:interp::
PyInterpreterState_Delete:void:::
PyInterpreterState_Delete:PyInterpreterState*:interp::
+PyInterpreterState_GetID:int64_t:::
+PyInterpreterState_GetID:PyInterpreterState*:interp::
+
PyInterpreterState_New:PyInterpreterState*:::
PyIter_Check:int:o:0:
@@ -921,6 +980,14 @@ PyNumber_Xor:PyObject*:o2:0:
PyObject_AsFileDescriptor:int:::
PyObject_AsFileDescriptor:PyObject*:o:0:
+PyOS_AfterFork:void:::
+
+PyOS_AfterFork_Child:void:::
+
+PyOS_AfterFork_Parent:void:::
+
+PyOS_BeforeFork:void:::
+
PyOS_FSPath:PyObject*::+1:
PyOS_FSPath:PyObject*:path:0:
@@ -1240,6 +1307,12 @@ PySet_Pop:PyObject*:set:0:
PySet_Size:int:::
PySet_Size:PyObject*:anyset:0:
+PySlice_AdjustIndices:Py_ssize_t:::
+PySlice_AdjustIndices:Py_ssize_t:length::
+PySlice_AdjustIndices:Py_ssize_t*:start::
+PySlice_AdjustIndices:Py_ssize_t*:stop::
+PySlice_AdjustIndices:Py_ssize_t*:step::
+
PySlice_Check:int:::
PySlice_Check:PyObject*:ob:0:
@@ -1248,6 +1321,12 @@ PySlice_New:PyObject*:start:0:
PySlice_New:PyObject*:stop:0:
PySlice_New:PyObject*:step:0:
+PySlice_Unpack:int:::
+PySlice_Unpack:PyObject*:slice:0:
+PySlice_Unpack:Py_ssize_t*:start::
+PySlice_Unpack:Py_ssize_t*:stop::
+PySlice_Unpack:Py_ssize_t*:step::
+
PyString_AS_STRING:const char*:::
PyString_AS_STRING:PyObject*:string:0:
@@ -1371,12 +1450,42 @@ PyThreadState_New:PyInterpreterState*:interp::
PyThreadState_Swap:PyThreadState*:::
PyThreadState_Swap:PyThreadState*:tstate::
+PyThread_tss_alloc:Py_tss_t*:::
+
+PyThread_tss_create:int:::
+PyThread_tss_create:Py_tss_t*:key::
+
+PyThread_tss_delete:void:::
+PyThread_tss_delete:Py_tss_t*:key::
+
+PyThread_tss_free:void:::
+PyThread_tss_free:Py_tss_t*:key::
+
+PyThread_tss_get:void*:::
+PyThread_tss_get:Py_tss_t*:key::
+
+PyThread_tss_is_created:int:::
+PyThread_tss_is_created:Py_tss_t*:key::
+
+PyThread_tss_set:int:::
+PyThread_tss_set:Py_tss_t*:key::
+PyThread_tss_set:void*:value::
+
PyTime_FromTime:PyObject*::+1:
PyTime_FromTime:int:hour::
PyTime_FromTime:int:minute::
PyTime_FromTime:int:second::
PyTime_FromTime:int:usecond::
+PyTraceMalloc_Track:int:::
+PyTraceMalloc_Track:unsigned int:domain::
+PyTraceMalloc_Track:uintptr_t:ptr::
+PyTraceMalloc_Track:size_t:size::
+
+PyTraceMalloc_Untrack:int:::
+PyTraceMalloc_Untrack:unsigned int:domain::
+PyTraceMalloc_Untrack:uintptr_t:ptr::
+
PyTuple_Check:int:::
PyTuple_Check:PyObject*:p:0:
diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst
index b0dccb3db1ddf4b9701a5a06fbf9593bed79ac5a..5dd14b1f7a60b95f0a2e073c2e7d557737dc77da 100644
--- a/Doc/distributing/index.rst
+++ b/Doc/distributing/index.rst
@@ -62,7 +62,7 @@ Key terms
locally.
.. _setuptools: https://setuptools.readthedocs.io/en/latest/
-.. _wheel: https://wheel.readthedocs.org
+.. _wheel: https://wheel.readthedocs.io/
Open source licensing and collaboration
=======================================
@@ -111,7 +111,7 @@ by invoking the ``pip`` module at the command line::
The Python Packaging User Guide includes more details on the `currently
recommended tools`_.
-.. _currently recommended tools: https://packaging.python.org/en/latest/current/#packaging-tool-recommendations
+.. _currently recommended tools: https://packaging.python.org/guides/tool-recommendations/#packaging-tool-recommendations
Reading the guide
=================
@@ -124,11 +124,11 @@ involved in creating a project:
* `Uploading the project to the Python Packaging Index`_
.. _Project structure: \
- https://packaging.python.org/en/latest/distributing/
+ https://packaging.python.org/tutorials/distributing-packages/
.. _Building and packaging the project: \
- https://packaging.python.org/en/latest/distributing/#packaging-your-project
+ https://packaging.python.org/tutorials/distributing-packages/#packaging-your-project
.. _Uploading the project to the Python Packaging Index: \
- https://packaging.python.org/en/latest/distributing/#uploading-your-project-to-pypi
+ https://packaging.python.org/tutorials/distributing-packages/#uploading-your-project-to-pypi
How do I...?
@@ -160,7 +160,7 @@ Python Packaging User Guide for more information and recommendations.
.. seealso::
`Python Packaging User Guide: Binary Extensions
- `__
+ `__
.. other topics:
diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst
index f147bb236a702ce00c3c129cc89b4bb87f66ce5b..8efeffb6376e136dec18baed66e7be9ec9df3818 100644
--- a/Doc/distutils/apiref.rst
+++ b/Doc/distutils/apiref.rst
@@ -183,8 +183,9 @@ the full reference.
| *sources* | list of source filenames, | a list of strings |
| | relative to the distribution | |
| | root (where the setup script | |
- | | lives), in Unix form (slash- | |
- | | separated) for portability. | |
+ | | lives), in Unix form | |
+ | | (slash-separated) for | |
+ | | portability. | |
| | Source files may be C, C++, | |
| | SWIG (.i), platform-specific | |
| | resource files, or whatever | |
@@ -285,6 +286,10 @@ the full reference.
See the :func:`setup` function for a list of keyword arguments accepted by the
Distribution constructor. :func:`setup` creates a Distribution instance.
+ .. versionchanged:: 3.7
+ :class:`~distutils.core.Distribution` now warns if ``classifiers``,
+ ``keywords`` and ``platforms`` fields are not specified as a list or
+ a string.
.. class:: Command
@@ -814,13 +819,13 @@ This module provides the :class:`UnixCCompiler` class, a subclass of
.. module:: distutils.msvccompiler
:synopsis: Microsoft Compiler
+.. XXX: This is *waaaaay* out of date!
This module provides :class:`MSVCCompiler`, an implementation of the abstract
:class:`CCompiler` class for Microsoft Visual Studio. Typically, extension
modules need to be compiled with the same compiler that was used to compile
Python. For Python 2.3 and earlier, the compiler was Visual Studio 6. For Python
-2.4 and 2.5, the compiler is Visual Studio .NET 2003. The AMD64 and Itanium
-binaries are created using the Platform SDK.
+2.4 and 2.5, the compiler is Visual Studio .NET 2003.
:class:`MSVCCompiler` will normally choose the right compiler, linker etc. on
its own. To override this choice, the environment variables *DISTUTILS_USE_SDK*
@@ -936,7 +941,7 @@ timestamp dependency analysis.
.. function:: newer_group(sources, target[, missing='error'])
Return true if *target* is out-of-date with respect to any file listed in
- *sources* In other words, if *target* exists and is newer than every file in
+ *sources*. In other words, if *target* exists and is newer than every file in
*sources*, return false; otherwise return true. *missing* controls what we do
when a source file is missing; the default (``'error'``) is to blow up with an
:exc:`OSError` from inside :func:`os.stat`; if it is ``'ignore'``, we silently
@@ -1086,19 +1091,16 @@ other utility module.
Return a string that identifies the current platform. This is used mainly to
distinguish platform-specific build directories and platform-specific built
- distributions. Typically includes the OS name and version and the architecture
- (as supplied by 'os.uname()'), although the exact information included depends
- on the OS; eg. for IRIX the architecture isn't particularly important (IRIX only
- runs on SGI hardware), but for Linux the kernel version isn't particularly
- important.
+ distributions. Typically includes the OS name and version and the
+ architecture (as supplied by 'os.uname()'), although the exact information
+ included depends on the OS; e.g., on Linux, the kernel version isn't
+ particularly important.
Examples of returned values:
* ``linux-i586``
* ``linux-alpha``
* ``solaris-2.6-sun4u``
- * ``irix-5.3``
- * ``irix64-6.2``
For non-POSIX platforms, currently just returns ``sys.platform``.
@@ -1565,8 +1567,8 @@ lines, and joining lines with backslashes.
+------------------+--------------------------------+---------+
| option name | description | default |
+==================+================================+=========+
- | *strip_comments* | strip from ``'#'`` to end-of- | true |
- | | line, as well as any | |
+ | *strip_comments* | strip from ``'#'`` to | true |
+ | | end-of-line, as well as any | |
| | whitespace leading up to the | |
| | ``'#'``\ ---unless it is | |
| | escaped by a backslash | |
diff --git a/Doc/distutils/builtdist.rst b/Doc/distutils/builtdist.rst
index bbd2a8ce831cee1daeb7dfe286e3b13259a75a0a..f1f3471261600ea38b04cb38e47f757d7359aa3d 100644
--- a/Doc/distutils/builtdist.rst
+++ b/Doc/distutils/builtdist.rst
@@ -21,7 +21,7 @@ specialty---writing code and creating source distributions---while an
intermediary species called *packagers* springs up to turn source distributions
into built distributions for as many platforms as there are packagers.
-Of course, the module developer could be his own packager; or the packager could
+Of course, the module developer could be their own packager; or the packager could
be a volunteer "out there" somewhere who has access to a platform which the
original developer does not; or it could be software periodically grabbing new
source distributions and turning them into built distributions for as many
@@ -63,9 +63,9 @@ distribution to generate: for example, ::
python setup.py bdist --format=zip
-would, when run on a Unix system, create :file:`Distutils-1.0.{plat}.zip`\
----again, this archive would be unpacked from the root directory to install the
-Distutils.
+would, when run on a Unix system, create
+:file:`Distutils-1.0.{plat}.zip`\ ---again, this archive would be unpacked
+from the root directory to install the Distutils.
The available formats for built distributions are:
@@ -351,8 +351,8 @@ installed, you can use a 32bit version of Windows to create 64bit extensions
and vice-versa.
To build for an alternate platform, specify the :option:`!--plat-name` option
-to the build command. Valid values are currently 'win32', 'win-amd64' and
-'win-ia64'. For example, on a 32bit version of Windows, you could execute::
+to the build command. Valid values are currently 'win32', and 'win-amd64'.
+For example, on a 32bit version of Windows, you could execute::
python setup.py build --plat-name=win-amd64
@@ -368,7 +368,7 @@ Python itself for the platform you are targeting - it is not possible from a
binary installation of Python (as the .lib etc file for other platforms are
not included.) In practice, this means the user of a 32 bit operating
system will need to use Visual Studio 2008 to open the
-:file:`PCBuild/PCbuild.sln` solution in the Python source tree and build the
+:file:`PCbuild/PCbuild.sln` solution in the Python source tree and build the
"x64" configuration of the 'pythoncore' project before cross-compiling
extensions is possible.
diff --git a/Doc/distutils/configfile.rst b/Doc/distutils/configfile.rst
index cd10a7fdf31513ef1a753e5557402f958d81e1cf..0874d05fe703a17a5cc9ac5b95162e470a8e6069 100644
--- a/Doc/distutils/configfile.rst
+++ b/Doc/distutils/configfile.rst
@@ -13,8 +13,8 @@ edit is a cheap and easy way to solicit it. Configuration files also let you
provide default values for any command option, which the installer can then
override either on the command-line or by editing the config file.
-The setup configuration file is a useful middle-ground between the setup script
----which, ideally, would be opaque to installers [#]_---and the command-line to
+The setup configuration file is a useful middle-ground between the setup
+script---which, ideally, would be opaque to installers [#]_---and the command-line to
the setup script, which is outside of your control and entirely up to the
installer. In fact, :file:`setup.cfg` (and any other Distutils configuration
files present on the target system) are processed after the contents of the
diff --git a/Doc/distutils/index.rst b/Doc/distutils/index.rst
index c565bcc562862490828bfb011464e2f9d488a7d1..d6f7640fcb6826a712cc19cc87b276f8edc1c156 100644
--- a/Doc/distutils/index.rst
+++ b/Doc/distutils/index.rst
@@ -22,7 +22,7 @@ very little overhead for build/release/install mechanics.
This guide only covers the basic tools for building and distributing
extensions that are provided as part of this version of Python. Third party
tools offer easier to use and more secure alternatives. Refer to the `quick
- recommendations section `__
+ recommendations section `__
in the Python Packaging User Guide for more information.
.. toctree::
diff --git a/Doc/distutils/introduction.rst b/Doc/distutils/introduction.rst
index a38555910361b85a565a43fe8c205415d74ae997..7721484fe737178988767223f315c8321900c8a7 100644
--- a/Doc/distutils/introduction.rst
+++ b/Doc/distutils/introduction.rst
@@ -94,7 +94,7 @@ containing your setup script :file:`setup.py`, and your module :file:`foo.py`.
The archive file will be named :file:`foo-1.0.tar.gz` (or :file:`.zip`), and
will unpack into a directory :file:`foo-1.0`.
-If an end-user wishes to install your :mod:`foo` module, all she has to do is
+If an end-user wishes to install your :mod:`foo` module, all they have to do is
download :file:`foo-1.0.tar.gz` (or :file:`.zip`), unpack it, and---from the
:file:`foo-1.0` directory---run ::
diff --git a/Doc/distutils/setupscript.rst b/Doc/distutils/setupscript.rst
index 21d569b65c3f52615001ae90239e706c6ef8c458..c1051d2e807e7946d928248ff659b26de5be2621 100644
--- a/Doc/distutils/setupscript.rst
+++ b/Doc/distutils/setupscript.rst
@@ -581,17 +581,19 @@ This information includes:
| | description of the | | |
| | package | | |
+----------------------+---------------------------+-----------------+--------+
-| ``long_description`` | longer description of the | long string | \(5) |
+| ``long_description`` | longer description of the | long string | \(4) |
| | package | | |
+----------------------+---------------------------+-----------------+--------+
-| ``download_url`` | location where the | URL | \(4) |
+| ``download_url`` | location where the | URL | |
| | package may be downloaded | | |
+----------------------+---------------------------+-----------------+--------+
-| ``classifiers`` | a list of classifiers | list of strings | \(4) |
+| ``classifiers`` | a list of classifiers | list of strings | (6)(7) |
+----------------------+---------------------------+-----------------+--------+
-| ``platforms`` | a list of platforms | list of strings | |
+| ``platforms`` | a list of platforms | list of strings | (6)(8) |
+----------------------+---------------------------+-----------------+--------+
-| ``license`` | license for the package | short string | \(6) |
+| ``keywords`` | a list of keywords | list of strings | (6)(8) |
++----------------------+---------------------------+-----------------+--------+
+| ``license`` | license for the package | short string | \(5) |
+----------------------+---------------------------+-----------------+--------+
Notes:
@@ -607,22 +609,30 @@ Notes:
provided, distutils lists it as the author in :file:`PKG-INFO`.
(4)
- These fields should not be used if your package is to be compatible with Python
- versions prior to 2.2.3 or 2.3. The list is available from the `PyPI website
- `_.
-
-(5)
The ``long_description`` field is used by PyPI when you are
:ref:`registering ` a package, to
:ref:`build its home page `.
-(6)
+(5)
The ``license`` field is a text indicating the license covering the
package where the license is not a selection from the "License" Trove
classifiers. See the ``Classifier`` field. Notice that
there's a ``licence`` distribution option which is deprecated but still
acts as an alias for ``license``.
+(6)
+ This field must be a list.
+
+(7)
+ The valid classifiers are listed on
+ `PyPI `_.
+
+(8)
+ To preserve backward compatibility, this field also accepts a string. If
+ you pass a comma-separated string ``'foo, bar'``, it will be converted to
+ ``['foo', 'bar']``, Otherwise, it will be converted to a list of one
+ string.
+
'short string'
A single line of text, not more than 200 characters.
@@ -650,7 +660,7 @@ information is sometimes used to indicate sub-releases. These are
1.0.1a2
the second alpha release of the first patch version of 1.0
-``classifiers`` are specified in a Python list::
+``classifiers`` must be specified in a list::
setup(...,
classifiers=[
@@ -671,6 +681,10 @@ information is sometimes used to indicate sub-releases. These are
],
)
+.. versionchanged:: 3.7
+ :class:`~distutils.core.setup` now warns when ``classifiers``, ``keywords``
+ or ``platforms`` fields are not specified as a list or a string.
+
.. _debug-setup-script:
Debugging the setup script
diff --git a/Doc/distutils/sourcedist.rst b/Doc/distutils/sourcedist.rst
index cc289c9b2551cbdf256a6b29f4db7b0db170456e..0ac8ef41ddc20fd710466cf6ff93cc166be488be 100644
--- a/Doc/distutils/sourcedist.rst
+++ b/Doc/distutils/sourcedist.rst
@@ -95,8 +95,9 @@ source distribution:
distributions, but in the future there will be a standard for testing Python
module distributions)
-* :file:`README.txt` (or :file:`README`), :file:`setup.py` (or whatever you
- called your setup script), and :file:`setup.cfg`
+* Any of the standard README files (:file:`README`, :file:`README.txt`,
+ or :file:`README.rst`), :file:`setup.py` (or whatever you called your setup
+ script), and :file:`setup.cfg`.
* all files that matches the ``package_data`` metadata.
See :ref:`distutils-installing-package-data`.
@@ -130,6 +131,9 @@ described above does not apply in this case.
:command:`sdist` will read a :file:`MANIFEST` file if no :file:`MANIFEST.in`
exists, like it used to do.
+.. versionchanged:: 3.7
+ :file:`README.rst` is now included in the list of distutils standard READMEs.
+
The manifest template has one command per line, where each command specifies a
set of files to include or exclude from the source distribution. For an
diff --git a/Doc/docutils.conf b/Doc/docutils.conf
deleted file mode 100644
index bda4f5dc2351dcb0fc42376809ef42a33d81392a..0000000000000000000000000000000000000000
--- a/Doc/docutils.conf
+++ /dev/null
@@ -1,2 +0,0 @@
-[restructuredtext parser]
-smartquotes-locales: ja: ""''
diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst
index 130f4a7321412f67e20a8880471c7a3de34b5921..b788a5575b3f2042c47a7dedf420bc0b10ce8d9c 100644
--- a/Doc/extending/extending.rst
+++ b/Doc/extending/extending.rst
@@ -27,7 +27,8 @@ your system setup; details are given in later chapters.
avoid writing C extensions and preserve portability to other implementations.
For example, if your use case is calling C library functions or system calls,
you should consider using the :mod:`ctypes` module or the `cffi
- `_ library rather than writing custom C code.
+ `_ library rather than writing
+ custom C code.
These modules let you write Python code to interface with C code and are more
portable between implementations of Python than writing and compiling a C
extension module.
@@ -544,8 +545,9 @@ or more format codes between parentheses. For example::
:c:func:`PyObject_CallObject` returns a Python object pointer: this is the return
value of the Python function. :c:func:`PyObject_CallObject` is
"reference-count-neutral" with respect to its arguments. In the example a new
-tuple was created to serve as the argument list, which is :c:func:`Py_DECREF`\
--ed immediately after the :c:func:`PyObject_CallObject` call.
+tuple was created to serve as the argument list, which is
+:c:func:`Py_DECREF`\ -ed immediately after the :c:func:`PyObject_CallObject`
+call.
The return value of :c:func:`PyObject_CallObject` is "new": either it is a brand
new object, or it is an existing object whose reference count has been
@@ -733,9 +735,9 @@ Philbrick (philbrick@hks.com)::
keywdarg_parrot(PyObject *self, PyObject *args, PyObject *keywds)
{
int voltage;
- char *state = "a stiff";
- char *action = "voom";
- char *type = "Norwegian Blue";
+ const char *state = "a stiff";
+ const char *action = "voom";
+ const char *type = "Norwegian Blue";
static char *kwlist[] = {"voltage", "state", "action", "type", NULL};
diff --git a/Doc/extending/index.rst b/Doc/extending/index.rst
index 533ed985f8290623ddff735548938af2940b92b5..0994e3e8627dfaed440585038bb7f29a93067e11 100644
--- a/Doc/extending/index.rst
+++ b/Doc/extending/index.rst
@@ -34,7 +34,7 @@ extensions for Python.
.. seealso::
- `Python Packaging User Guide: Binary Extensions `_
+ `Python Packaging User Guide: Binary Extensions `_
The Python Packaging User Guide not only covers several available
tools that simplify the creation of binary extensions, but also
discusses the various reasons why creating an extension module may be
diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst
index b55796421c0fbce9e2c568aa574090c7623529c4..d0d2ec1f88207c9683350d7110f7e2a56fd5692c 100644
--- a/Doc/extending/newtypes.rst
+++ b/Doc/extending/newtypes.rst
@@ -263,11 +263,11 @@ in the instance. A variety of primitive C types are supported, and access may
be read-only or read-write. The structures in the table are defined as::
typedef struct PyMemberDef {
- char *name;
- int type;
- int offset;
- int flags;
- char *doc;
+ const char *name;
+ int type;
+ int offset;
+ int flags;
+ const char *doc;
} PyMemberDef;
For each entry in the table, a :term:`descriptor` will be constructed and added to the
@@ -487,9 +487,9 @@ Here is a toy ``tp_call`` implementation::
newdatatype_call(newdatatypeobject *self, PyObject *args, PyObject *kwds)
{
PyObject *result;
- char *arg1;
- char *arg2;
- char *arg3;
+ const char *arg1;
+ const char *arg2;
+ const char *arg3;
if (!PyArg_ParseTuple(args, "sss:call", &arg1, &arg2, &arg3)) {
return NULL;
diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst
index 1bd800b1a81babcf638fa3cf910cb66603525e2e..e2d63a0323da661b043e6672b5c4e3e2df79df27 100644
--- a/Doc/faq/design.rst
+++ b/Doc/faq/design.rst
@@ -2,6 +2,11 @@
Design and History FAQ
======================
+.. only:: html
+
+ .. contents::
+
+
Why does Python use indentation for grouping of statements?
-----------------------------------------------------------
@@ -210,24 +215,25 @@ objects using the ``for`` statement. For example, :term:`file objects
Why does Python use methods for some functionality (e.g. list.index()) but functions for other (e.g. len(list))?
----------------------------------------------------------------------------------------------------------------
-The major reason is history. Functions were used for those operations that were
-generic for a group of types and which were intended to work even for objects
-that didn't have methods at all (e.g. tuples). It is also convenient to have a
-function that can readily be applied to an amorphous collection of objects when
-you use the functional features of Python (``map()``, ``zip()`` et al).
+As Guido said:
-In fact, implementing ``len()``, ``max()``, ``min()`` as a built-in function is
-actually less code than implementing them as methods for each type. One can
-quibble about individual cases but it's a part of Python, and it's too late to
-make such fundamental changes now. The functions have to remain to avoid massive
-code breakage.
+ (a) For some operations, prefix notation just reads better than
+ postfix -- prefix (and infix!) operations have a long tradition in
+ mathematics which likes notations where the visuals help the
+ mathematician thinking about a problem. Compare the easy with which we
+ rewrite a formula like x*(a+b) into x*a + x*b to the clumsiness of
+ doing the same thing using a raw OO notation.
-.. XXX talk about protocols?
+ (b) When I read code that says len(x) I *know* that it is asking for
+ the length of something. This tells me two things: the result is an
+ integer, and the argument is some kind of container. To the contrary,
+ when I read x.len(), I have to already know that x is some kind of
+ container implementing an interface or inheriting from a class that
+ has a standard len(). Witness the confusion we occasionally have when
+ a class that is not implementing a mapping has a get() or keys()
+ method, or something that isn't a file has a write() method.
-.. note::
-
- For string operations, Python has moved from external functions (the
- ``string`` module) to methods. However, ``len()`` is still a function.
+ -- https://mail.python.org/pipermail/python-3000/2006-November/004643.html
Why is join() a string method instead of a list or tuple method?
@@ -343,7 +349,7 @@ each Python stack frame. Also, extensions can call back into Python at almost
random moments. Therefore, a complete threads implementation requires thread
support for C.
-Answer 2: Fortunately, there is `Stackless Python `_,
+Answer 2: Fortunately, there is `Stackless Python `_,
which has a completely redesigned interpreter loop that avoids the C stack.
@@ -465,10 +471,10 @@ you can always change a list's elements. Only immutable elements can be used as
dictionary keys, and hence only tuples and not lists can be used as keys.
-How are lists implemented?
---------------------------
+How are lists implemented in CPython?
+-------------------------------------
-Python's lists are really variable-length arrays, not Lisp-style linked lists.
+CPython's lists are really variable-length arrays, not Lisp-style linked lists.
The implementation uses a contiguous array of references to other objects, and
keeps a pointer to this array and the array's length in a list head structure.
@@ -481,10 +487,10 @@ when the array must be grown, some extra space is allocated so the next few
times don't require an actual resize.
-How are dictionaries implemented?
----------------------------------
+How are dictionaries implemented in CPython?
+--------------------------------------------
-Python's dictionaries are implemented as resizable hash tables. Compared to
+CPython's dictionaries are implemented as resizable hash tables. Compared to
B-trees, this gives better performance for lookup (the most common operation by
far) under most circumstances, and the implementation is simpler.
@@ -495,11 +501,7 @@ on the key and a per-process seed; for example, "Python" could hash to
to 1142331976. The hash code is then used to calculate a location in an
internal array where the value will be stored. Assuming that you're storing
keys that all have different hash values, this means that dictionaries take
-constant time -- O(1), in computer science notation -- to retrieve a key. It
-also means that no sorted order of the keys is maintained, and traversing the
-array as the ``.keys()`` and ``.items()`` do will output the dictionary's
-content in some arbitrary jumbled order that can change with every invocation of
-a program.
+constant time -- O(1), in Big-O notation -- to retrieve a key.
Why must dictionary keys be immutable?
@@ -526,7 +528,7 @@ Some unacceptable solutions that have been proposed:
mydict = {[1, 2]: '12'}
print(mydict[[1, 2]])
- would raise a KeyError exception because the id of the ``[1, 2]`` used in the
+ would raise a :exc:`KeyError` exception because the id of the ``[1, 2]`` used in the
second line differs from that in the first line. In other words, dictionary
keys should be compared using ``==``, not using :keyword:`is`.
diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst
index fd04a83df33c3df29d2ce10896d1021e16ec4df7..74e1af6ef24de1719f7969d3b833323b7feaa12e 100644
--- a/Doc/faq/extending.rst
+++ b/Doc/faq/extending.rst
@@ -63,7 +63,7 @@ How can I execute arbitrary Python statements from C?
The highest-level function to do this is :c:func:`PyRun_SimpleString` which takes
a single string argument to be executed in the context of the module
``__main__`` and returns ``0`` for success and ``-1`` when an exception occurred
-(including ``SyntaxError``). If you want more control, use
+(including :exc:`SyntaxError`). If you want more control, use
:c:func:`PyRun_String`; see the source for :c:func:`PyRun_SimpleString` in
``Python/pythonrun.c``.
@@ -277,7 +277,7 @@ However sometimes you have to run the embedded Python interpreter in the same
thread as your rest application and you can't allow the
:c:func:`PyRun_InteractiveLoop` to stop while waiting for user input. The one
solution then is to call :c:func:`PyParser_ParseString` and test for ``e.error``
-equal to ``E_EOF``, which means the input is incomplete). Here's a sample code
+equal to ``E_EOF``, which means the input is incomplete. Here's a sample code
fragment, untested, inspired by code from Alex Farber::
#include
diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst
index 4e842f5366ef0e9aa6c948a4c7c2985a9626f589..90fd69e72960c0b876f50dd649c11ce8af3d8fe4 100644
--- a/Doc/faq/general.rst
+++ b/Doc/faq/general.rst
@@ -272,7 +272,7 @@ The Python project's infrastructure is located all over the world.
`www.python.org `_ is graciously hosted by `Rackspace
`_, with CDN caching provided by `Fastly
`_. `Upfront Systems
-`_ hosts `bugs.python.org
+`_ hosts `bugs.python.org
`_. Many other Python services like `the Wiki
`_ are hosted by `Oregon State
University Open Source Lab `_.
@@ -306,17 +306,19 @@ usually around 18 months between major releases.
The developers issue "bugfix" releases of older versions, so the stability of
existing releases gradually improves. Bugfix releases, indicated by a third
-component of the version number (e.g. 2.5.3, 2.6.2), are managed for stability;
+component of the version number (e.g. 3.5.3, 3.6.2), are managed for stability;
only fixes for known problems are included in a bugfix release, and it's
guaranteed that interfaces will remain the same throughout a series of bugfix
releases.
The latest stable releases can always be found on the `Python download page
-`_. There are two recommended production-ready
-versions at this point in time, because at the moment there are two branches of
-stable releases: 2.x and 3.x. Python 3.x may be less useful than 2.x, since
-currently there is more third party software available for Python 2 than for
-Python 3. Python 2 code will generally not run unchanged in Python 3.
+`_. There are two production-ready version
+of Python: 2.x and 3.x, but the recommended one at this times is Python 3.x.
+Although Python 2.x is still widely used, `it will not be
+maintained after January 1, 2020 `_.
+Python 2.x was known for having more third-party libraries available, however,
+by the time of this writing, most of the widely used libraries support Python 3.x,
+and some are even dropping the Python 2.x support.
How many people are using Python?
diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst
index 38e1796267ff3a75d324f8f70a40d935d8e8d90e..4f9979bf55ed3a2df8bd0e366ca5b2ed39923d7c 100644
--- a/Doc/faq/gui.rst
+++ b/Doc/faq/gui.rst
@@ -43,7 +43,7 @@ number of platforms, with Windows, Mac OS X, GTK, X11, all listed as
current stable targets. Language bindings are available for a number
of languages including Python, Perl, Ruby, etc.
-wxPython (http://www.wxpython.org) is the Python binding for
+`wxPython `_ is the Python binding for
wxwidgets. While it often lags slightly behind the official wxWidgets
releases, it also offers a number of features via pure Python
extensions that are not available in other language bindings. There
@@ -72,9 +72,9 @@ Gtk+
The `GObject introspection bindings `_
for Python allow you to write GTK+ 3 applications. There is also a
-`Python GTK+ 3 Tutorial `_.
+`Python GTK+ 3 Tutorial `_.
-The older PyGtk bindings for the `Gtk+ 2 toolkit `_ have
+The older PyGtk bindings for the `Gtk+ 2 toolkit `_ have
been implemented by James Henstridge; see .
Kivy
diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst
index 1acfc26801ca22f0c3cb2257ffc6e73ee8a550f8..ab92a879a8851617accefcf9c4cf11f5442054a5 100644
--- a/Doc/faq/library.rst
+++ b/Doc/faq/library.rst
@@ -421,7 +421,7 @@ Python program effectively only uses one CPU, due to the insistence that
Back in the days of Python 1.5, Greg Stein actually implemented a comprehensive
patch set (the "free threading" patches) that removed the GIL and replaced it
with fine-grained locking. Adam Olsen recently did a similar experiment
-in his `python-safethread `_
+in his `python-safethread `_
project. Unfortunately, both experiments exhibited a sharp drop in single-thread
performance (at least 30% slower), due to the amount of fine-grained locking
necessary to compensate for the removal of the GIL.
diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst
index 7476ce11f4f141189e241ddb22640d67b1173468..fd720c1a304b0f3f5e003a8b561c6d757aeb1bd4 100644
--- a/Doc/faq/programming.rst
+++ b/Doc/faq/programming.rst
@@ -71,6 +71,11 @@ length, whether variable names are well-formed according to your coding
standard, whether declared interfaces are fully implemented, and more.
https://docs.pylint.org/ provides a full list of Pylint's features.
+Static type checkers such as `Mypy `_,
+`Pyre `_, and
+`Pytype `_ can check type hints in Python
+source code.
+
How can I create a stand-alone binary from a Python script?
-----------------------------------------------------------
@@ -100,7 +105,7 @@ which don't. One is Thomas Heller's py2exe (Windows only) at
http://www.py2exe.org/
-Another tool is Anthony Tuininga's `cx_Freeze `_.
+Another tool is Anthony Tuininga's `cx_Freeze `_.
Are there coding standards or a style guide for Python programs?
@@ -371,8 +376,8 @@ compute, a common technique is to cache the parameters and the resulting value
of each call to the function, and return the cached value if the same value is
requested again. This is called "memoizing", and can be implemented like this::
- # Callers will never provide a third parameter for this function.
- def expensive(arg1, arg2, _cache={}):
+ # Callers can only provide two parameters and optionally pass _cache by keyword
+ def expensive(arg1, arg2, *, _cache={}):
if (arg1, arg2) in _cache:
return _cache[(arg1, arg2)]
@@ -1315,11 +1320,6 @@ that final assignment still results in an error, because tuples are immutable.
Dictionaries
============
-How can I get a dictionary to store and display its keys in a consistent order?
--------------------------------------------------------------------------------
-
-Use :class:`collections.OrderedDict`.
-
I want to do a complicated sort: can you do a Schwartzian Transform in Python?
------------------------------------------------------------------------------
@@ -1849,7 +1849,7 @@ containing statements like ::
will continue to work with the old version of the imported objects. If the
module contains class definitions, existing class instances will *not* be
updated to use the new class definition. This can result in the following
-paradoxical behaviour:
+paradoxical behaviour::
>>> import importlib
>>> import cls
@@ -1860,7 +1860,7 @@ paradoxical behaviour:
False
The nature of the problem is made clear if you print out the "identity" of the
-class objects:
+class objects::
>>> hex(id(c.__class__))
'0x7352a0'
diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst
index f1839845832c0f1db7707bca6ea2b3bd86ba4047..a46d4c1725ac97e147a78cea60e9dae1a1266f21 100644
--- a/Doc/faq/windows.rst
+++ b/Doc/faq/windows.rst
@@ -23,23 +23,10 @@ This is not necessarily a straightforward question. If you are already familiar
with running programs from the Windows command line then everything will seem
obvious; otherwise, you might need a little more guidance.
-.. sidebar:: |Python Development on XP|_
- :subtitle: `Python Development on XP`_
-
- This series of screencasts aims to get you up and running with Python on
- Windows XP. The knowledge is distilled into 1.5 hours and will get you up
- and running with the right Python distribution, coding in your choice of IDE,
- and debugging and writing solid code with unit-tests.
-
-.. |Python Development on XP| image:: python-video-icon.png
-.. _`Python Development on XP`:
- http://showmedo.com/videotutorials/series?name=pythonOzsvaldPyNewbieSeries
-
Unless you use some sort of integrated development environment, you will end up
*typing* Windows commands into what is variously referred to as a "DOS window"
or "Command prompt window". Usually you can create such a window from your
-Start menu; under Windows 7 the menu selection is :menuselection:`Start -->
-Programs --> Accessories --> Command Prompt`. You should be able to recognize
+search bar by searching for ``cmd``. You should be able to recognize
when you have started such a window because you will see a Windows "command
prompt", which usually looks like this:
@@ -64,19 +51,19 @@ compiles it into bytecodes, and then executes the bytecodes to run your
program. So, how do you arrange for the interpreter to handle your Python?
First, you need to make sure that your command window recognises the word
-"python" as an instruction to start the interpreter. If you have opened a
-command window, you should try entering the command ``python`` and hitting
+"py" as an instruction to start the interpreter. If you have opened a
+command window, you should try entering the command ``py`` and hitting
return:
.. code-block:: doscon
- C:\Users\YourName> python
+ C:\Users\YourName> py
You should then see something like:
.. code-block:: pycon
- Python 3.3.0 (v3.3.0:bd8afb90ebf2, Sep 29 2012, 10:55:48) [MSC v.1600 32 bit (Intel)] on win32
+ Python 3.6.4 (v3.6.4:d48eceb, Dec 19 2017, 06:04:45) [MSC v.1900 32 bit (Intel)] on win32
Type "help", "copyright", "credits" or "license" for more information.
>>>
@@ -93,64 +80,33 @@ by entering a few expressions of your choice and seeing the results:
'HelloHelloHello'
Many people use the interactive mode as a convenient yet highly programmable
-calculator. When you want to end your interactive Python session, hold the :kbd:`Ctrl`
-key down while you enter a :kbd:`Z`, then hit the ":kbd:`Enter`" key to get back to your
-Windows command prompt.
+calculator. When you want to end your interactive Python session,
+call the :func:`exit` function or hold the :kbd:`Ctrl` key down
+while you enter a :kbd:`Z`, then hit the ":kbd:`Enter`" key to get
+back to your Windows command prompt.
You may also find that you have a Start-menu entry such as :menuselection:`Start
---> Programs --> Python 3.3 --> Python (command line)` that results in you
+--> Programs --> Python 3.x --> Python (command line)` that results in you
seeing the ``>>>`` prompt in a new window. If so, the window will disappear
-after you enter the :kbd:`Ctrl-Z` character; Windows is running a single "python"
+after you call the :func:`exit` function or enter the :kbd:`Ctrl-Z`
+character; Windows is running a single "python"
command in the window, and closes it when you terminate the interpreter.
-If the ``python`` command, instead of displaying the interpreter prompt ``>>>``,
-gives you a message like::
-
- 'python' is not recognized as an internal or external command, operable program or batch file.
-
-.. sidebar:: |Adding Python to DOS Path|_
- :subtitle: `Adding Python to DOS Path`_
-
- Python is not added to the DOS path by default. This screencast will walk
- you through the steps to add the correct entry to the `System Path`, allowing
- Python to be executed from the command-line by all users.
-
-.. |Adding Python to DOS Path| image:: python-video-icon.png
-.. _`Adding Python to DOS Path`:
- http://showmedo.com/videotutorials/video?name=960000&fromSeriesID=96
-
-
-or::
-
- Bad command or filename
-
-then you need to make sure that your computer knows where to find the Python
-interpreter. To do this you will have to modify a setting called PATH, which is
-a list of directories where Windows will look for programs.
-
-You should arrange for Python's installation directory to be added to the PATH
-of every command window as it starts. If you installed Python fairly recently
-then the command ::
-
- dir C:\py*
+Now that we know the ``py`` command is recognized, you can give your
+Python script to it. You'll have to give either an absolute or a
+relative path to the Python script. Let's say your Python script is
+located in your desktop and is named ``hello.py``, and your command
+prompt is nicely opened in your home directory so you're seeing something
+similar to::
-will probably tell you where it is installed; the usual location is something
-like ``C:\Python33``. Otherwise you will be reduced to a search of your whole
-disk ... use :menuselection:`Tools --> Find` or hit the :guilabel:`Search`
-button and look for "python.exe". Supposing you discover that Python is
-installed in the ``C:\Python33`` directory (the default at the time of writing),
-you should make sure that entering the command ::
+ C:\Users\YourName>
- c:\Python33\python
+So now you'll ask the ``py`` command to give your script to Python by
+typing ``py`` followed by your script path::
-starts up the interpreter as above (and don't forget you'll need a ":kbd:`Ctrl-Z`" and
-an ":kbd:`Enter`" to get out of it). Once you have verified the directory, you can
-add it to the system path to make it easier to start Python by just running
-the ``python`` command. This is currently an option in the installer as of
-CPython 3.3.
-More information about environment variables can be found on the
-:ref:`Using Python on Windows ` page.
+ C:\Users\YourName> py Desktop\hello.py
+ hello
How do I make Python scripts executable?
----------------------------------------
@@ -182,8 +138,8 @@ offender.
How do I make an executable from a Python script?
-------------------------------------------------
-See http://cx-freeze.sourceforge.net/ for a distutils extension that allows you
-to create console and GUI executables from Python code.
+See `cx_Freeze `_ for a distutils extension
+that allows you to create console and GUI executables from Python code.
`py2exe `_, the most popular extension for building
Python 2.x-based executables, does not yet support Python 3 but a version that
does is in development.
@@ -324,36 +280,3 @@ How do I check for a keypress without blocking?
Use the msvcrt module. This is a standard Windows-specific extension module.
It defines a function ``kbhit()`` which checks whether a keyboard hit is
present, and ``getch()`` which gets one character without echoing it.
-
-
-How do I emulate os.kill() in Windows?
---------------------------------------
-
-Prior to Python 2.7 and 3.2, to terminate a process, you can use :mod:`ctypes`:
-
-.. code-block:: python
-
- import ctypes
-
- def kill(pid):
- """kill function for Win32"""
- kernel32 = ctypes.windll.kernel32
- handle = kernel32.OpenProcess(1, 0, pid)
- return (0 != kernel32.TerminateProcess(handle, 0))
-
-In 2.7 and 3.2, :func:`os.kill` is implemented similar to the above function,
-with the additional feature of being able to send :kbd:`Ctrl+C` and :kbd:`Ctrl+Break`
-to console subprocesses which are designed to handle those signals. See
-:func:`os.kill` for further details.
-
-How do I extract the downloaded documentation on Windows?
----------------------------------------------------------
-
-Sometimes, when you download the documentation package to a Windows machine
-using a web browser, the file extension of the saved file ends up being .EXE.
-This is a mistake; the extension should be .TGZ.
-
-Simply rename the downloaded file to have the .TGZ extension, and WinZip will be
-able to handle it. (If your copy of WinZip doesn't, get a newer one from
-https://www.winzip.com.)
-
diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index 6dbf0c398ddb0ba94a5dd03f0c3040fc260f4136..b8d98dd34c13091ad4d6ffaa19b5dc23031018cc 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -14,8 +14,9 @@ Glossary
``...``
The default Python prompt of the interactive shell when entering code for
- an indented code block or within a pair of matching left and right
- delimiters (parentheses, square brackets or curly braces).
+ an indented code block, when within a pair of matching left and right
+ delimiters (parentheses, square brackets, curly braces or triple quotes),
+ or after specifying a decorator.
2to3
A tool that tries to convert Python 2.x code to Python 3.x code by
@@ -94,7 +95,7 @@ Glossary
that it contains :keyword:`yield` expressions for producing a series of
values usable in an :keyword:`async for` loop.
- Usually refers to a asynchronous generator function, but may refer to an
+ Usually refers to an asynchronous generator function, but may refer to an
*asynchronous generator iterator* in some contexts. In cases where the
intended meaning isn't clear, using the full terms avoids ambiguity.
@@ -107,8 +108,8 @@ Glossary
This is an :term:`asynchronous iterator` which when called using the
:meth:`__anext__` method returns an awaitable object which will execute
- that the body of the asynchronous generator function until the
- next :keyword:`yield` expression.
+ the body of the asynchronous generator function until the next
+ :keyword:`yield` expression.
Each :keyword:`yield` temporarily suspends processing, remembering the
location execution state (including local variables and pending
@@ -122,10 +123,10 @@ Glossary
:meth:`__aiter__` method. Introduced by :pep:`492`.
asynchronous iterator
- An object that implements :meth:`__aiter__` and :meth:`__anext__`
+ An object that implements the :meth:`__aiter__` and :meth:`__anext__`
methods. ``__anext__`` must return an :term:`awaitable` object.
- :keyword:`async for` resolves awaitable returned from asynchronous
- iterator's :meth:`__anext__` method until it raises
+ :keyword:`async for` resolves the awaitables returned by an asynchronous
+ iterator's :meth:`__anext__` method until it raises a
:exc:`StopAsyncIteration` exception. Introduced by :pep:`492`.
attribute
@@ -140,7 +141,7 @@ Glossary
BDFL
Benevolent Dictator For Life, a.k.a. `Guido van Rossum
- `_, Python's creator.
+ `_, Python's creator.
binary file
A :term:`file object` able to read and write
@@ -199,7 +200,7 @@ Glossary
``int(3.15)`` converts the floating point number to the integer ``3``, but
in ``3+4.5``, each argument is of a different type (one int, one float),
and both must be converted to the same type before they can be added or it
- will raise a ``TypeError``. Without coercion, all arguments of even
+ will raise a :exc:`TypeError`. Without coercion, all arguments of even
compatible types would have to be normalized to the same value by the
programmer, e.g., ``float(3)+4.5`` rather than just ``3+4.5``.
@@ -388,7 +389,7 @@ Glossary
An :term:`annotation` of a function parameter or return value.
Function annotations are usually used for
- :term:`type hints `: for example this function is expected to take two
+ :term:`type hints `: for example, this function is expected to take two
:class:`int` arguments and is also expected to have an :class:`int`
return value::
@@ -483,6 +484,12 @@ Glossary
is believed that overcoming this performance issue would make the
implementation much more complicated and therefore costlier to maintain.
+
+ hash-based pyc
+ A bytecode cache file that uses the hash rather than the last-modified
+ time of the corresponding source file to determine its validity. See
+ :ref:`pyc-invalidation`.
+
hashable
An object is *hashable* if it has a hash value which never changes during
its lifetime (it needs a :meth:`__hash__` method), and can be compared to
@@ -636,7 +643,7 @@ Glossary
list
A built-in Python :term:`sequence`. Despite its name it is more akin
to an array in other languages than to a linked list since access to
- elements are O(1).
+ elements is O(1).
list comprehension
A compact way to process all or part of the elements in a sequence and
@@ -1005,7 +1012,7 @@ Glossary
struct sequence
A tuple with named elements. Struct sequences expose an interface similar
- to :term:`named tuple` in that elements can either be accessed either by
+ to :term:`named tuple` in that elements can be accessed either by
index or as an attribute. However, they do not have any of the named tuple
methods like :meth:`~collections.somenamedtuple._make` or
:meth:`~collections.somenamedtuple._asdict`. Examples of struct sequences
diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst
index 77b2aae088c3e9f4beee06f18a4eb2983ad4513a..695fbb1be192b2e1408ec2a0ebd3cd3a951033e3 100644
--- a/Doc/howto/clinic.rst
+++ b/Doc/howto/clinic.rst
@@ -1064,7 +1064,7 @@ Currently Argument Clinic supports only a few return converters:
DecodeFSDefault
None of these take parameters. For the first three, return -1 to indicate
-error. For ``DecodeFSDefault``, the return type is ``char *``; return a NULL
+error. For ``DecodeFSDefault``, the return type is ``const char *``; return a NULL
pointer to indicate an error.
(There's also an experimental ``NoneType`` converter, which lets you
diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst
index d5c07714aded6ef21a6245027f5fb8b2cdece6b1..cc4b4785b12290dcb0c500d418c6ac1b1fb8aec0 100644
--- a/Doc/howto/curses.rst
+++ b/Doc/howto/curses.rst
@@ -543,7 +543,7 @@ learn more about submitting patches to Python.
* `Writing Programs with NCURSES `_:
a lengthy tutorial for C programmers.
-* `The ncurses man page `_
+* `The ncurses man page `_
* `The ncurses FAQ `_
* `"Use curses... don't swear" `_:
video of a PyCon 2013 talk on controlling terminals using curses or Urwid.
diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst
index 5e85a9aa6594e458a7ef8470f2248f0133827d6b..51520b72032867ed2f3a69f8f508da2e4ca345b1 100644
--- a/Doc/howto/descriptor.rst
+++ b/Doc/howto/descriptor.rst
@@ -11,7 +11,7 @@ Abstract
--------
Defines descriptors, summarizes the protocol, and shows how descriptors are
-called. Examines a custom descriptor and several built-in python descriptors
+called. Examines a custom descriptor and several built-in Python descriptors
including functions, properties, static methods, and class methods. Shows how
each works by giving a pure Python equivalent and a sample application.
@@ -48,11 +48,11 @@ a flexible set of new tools for everyday Python programs.
Descriptor Protocol
-------------------
-``descr.__get__(self, obj, type=None) --> value``
+``descr.__get__(self, obj, type=None) -> value``
-``descr.__set__(self, obj, value) --> None``
+``descr.__set__(self, obj, value) -> None``
-``descr.__delete__(self, obj) --> None``
+``descr.__delete__(self, obj) -> None``
That is all there is to it. Define any of these methods and an object is
considered a descriptor and can override default behavior upon being looked up
@@ -275,7 +275,7 @@ variable name.
To support method calls, functions include the :meth:`__get__` method for
binding methods during attribute access. This means that all functions are
non-data descriptors which return bound methods when they are invoked from an
-object. In pure python, it works like this::
+object. In pure Python, it works like this::
class Function(object):
. . .
diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst
index 40601812a77cb583c8f381ef8b2cba0512be82d8..2efe4537e288d674a976c8021a50d1c7d1e151e2 100644
--- a/Doc/howto/functional.rst
+++ b/Doc/howto/functional.rst
@@ -198,7 +198,7 @@ for it.
You can experiment with the iteration interface manually:
- >>> L = [1,2,3]
+ >>> L = [1, 2, 3]
>>> it = iter(L)
>>> it #doctest: +ELLIPSIS
<...iterator object at ...>
@@ -229,7 +229,7 @@ iterator. These two statements are equivalent::
Iterators can be materialized as lists or tuples by using the :func:`list` or
:func:`tuple` constructor functions:
- >>> L = [1,2,3]
+ >>> L = [1, 2, 3]
>>> iterator = iter(L)
>>> t = tuple(iterator)
>>> t
@@ -238,10 +238,10 @@ Iterators can be materialized as lists or tuples by using the :func:`list` or
Sequence unpacking also supports iterators: if you know an iterator will return
N elements, you can unpack them into an N-tuple:
- >>> L = [1,2,3]
+ >>> L = [1, 2, 3]
>>> iterator = iter(L)
- >>> a,b,c = iterator
- >>> a,b,c
+ >>> a, b, c = iterator
+ >>> a, b, c
(1, 2, 3)
Built-in functions such as :func:`max` and :func:`min` can take a single
@@ -273,23 +273,24 @@ dictionary's keys::
>>> m = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6,
... 'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12}
- >>> for key in m: #doctest: +SKIP
+ >>> for key in m:
... print(key, m[key])
- Mar 3
+ Jan 1
Feb 2
- Aug 8
- Sep 9
+ Mar 3
Apr 4
+ May 5
Jun 6
Jul 7
- Jan 1
- May 5
+ Aug 8
+ Sep 9
+ Oct 10
Nov 11
Dec 12
- Oct 10
-Note that the order is essentially random, because it's based on the hash
-ordering of the objects in the dictionary.
+Note that starting with Python 3.7, dictionary iteration order is guaranteed
+to be the same as the insertion order. In earlier versions, the behaviour was
+unspecified and could vary between implementations.
Applying :func:`iter` to a dictionary always loops over the keys, but
dictionaries have methods that return other iterators. If you want to iterate
@@ -301,8 +302,8 @@ The :func:`dict` constructor can accept an iterator that returns a finite stream
of ``(key, value)`` tuples:
>>> L = [('Italy', 'Rome'), ('France', 'Paris'), ('US', 'Washington DC')]
- >>> dict(iter(L)) #doctest: +SKIP
- {'Italy': 'Rome', 'US': 'Washington DC', 'France': 'Paris'}
+ >>> dict(iter(L))
+ {'Italy': 'Rome', 'France': 'Paris', 'US': 'Washington DC'}
Files also support iteration by calling the :meth:`~io.TextIOBase.readline`
method until there are no more lines in the file. This means you can read each
@@ -410,7 +411,7 @@ lengths of all the sequences. If you have two lists of length 3, the output
list is 9 elements long:
>>> seq1 = 'abc'
- >>> seq2 = (1,2,3)
+ >>> seq2 = (1, 2, 3)
>>> [(x, y) for x in seq1 for y in seq2] #doctest: +NORMALIZE_WHITESPACE
[('a', 1), ('a', 2), ('a', 3),
('b', 1), ('b', 2), ('b', 3),
@@ -478,7 +479,7 @@ Here's a sample usage of the ``generate_ints()`` generator:
File "stdin", line 2, in generate_ints
StopIteration
-You could equally write ``for i in generate_ints(5)``, or ``a,b,c =
+You could equally write ``for i in generate_ints(5)``, or ``a, b, c =
generate_ints(3)``.
Inside a generator function, ``return value`` causes ``StopIteration(value)``
@@ -694,17 +695,17 @@ truth values of an iterable's contents. :func:`any` returns ``True`` if any ele
in the iterable is a true value, and :func:`all` returns ``True`` if all of the
elements are true values:
- >>> any([0,1,0])
+ >>> any([0, 1, 0])
True
- >>> any([0,0,0])
+ >>> any([0, 0, 0])
False
- >>> any([1,1,1])
+ >>> any([1, 1, 1])
True
- >>> all([0,1,0])
+ >>> all([0, 1, 0])
False
- >>> all([0,0,0])
+ >>> all([0, 0, 0])
False
- >>> all([1,1,1])
+ >>> all([1, 1, 1])
True
@@ -763,7 +764,7 @@ which defaults to 0, and the interval between numbers, which defaults to 1::
a provided iterable and returns a new iterator that returns its elements from
first to last. The new iterator will repeat these elements infinitely. ::
- itertools.cycle([1,2,3,4,5]) =>
+ itertools.cycle([1, 2, 3, 4, 5]) =>
1, 2, 3, 4, 5, 1, 2, 3, 4, 5, ...
:func:`itertools.repeat(elem, [n]) ` returns the provided
@@ -874,7 +875,7 @@ iterable's results. ::
iterators and returns only those elements of *data* for which the corresponding
element of *selectors* is true, stopping whenever either one is exhausted::
- itertools.compress([1,2,3,4,5], [True, True, False, False, True]) =>
+ itertools.compress([1, 2, 3, 4, 5], [True, True, False, False, True]) =>
1, 2, 5
@@ -1034,7 +1035,7 @@ first calculation. ::
Traceback (most recent call last):
...
TypeError: reduce() of empty sequence with no initial value
- >>> functools.reduce(operator.mul, [1,2,3], 1)
+ >>> functools.reduce(operator.mul, [1, 2, 3], 1)
6
>>> functools.reduce(operator.mul, [], 1)
1
@@ -1044,9 +1045,9 @@ elements of the iterable. This case is so common that there's a special
built-in called :func:`sum` to compute it:
>>> import functools, operator
- >>> functools.reduce(operator.add, [1,2,3,4], 0)
+ >>> functools.reduce(operator.add, [1, 2, 3, 4], 0)
10
- >>> sum([1,2,3,4])
+ >>> sum([1, 2, 3, 4])
10
>>> sum([])
0
@@ -1056,11 +1057,11 @@ write the obvious :keyword:`for` loop::
import functools
# Instead of:
- product = functools.reduce(operator.mul, [1,2,3], 1)
+ product = functools.reduce(operator.mul, [1, 2, 3], 1)
# You can write:
product = 1
- for i in [1,2,3]:
+ for i in [1, 2, 3]:
product *= i
A related function is :func:`itertools.accumulate(iterable, func=operator.add)
@@ -1068,10 +1069,10 @@ A related function is :func:`itertools.accumulate(iterable, func=operator.add)
returning only the final result, :func:`accumulate` returns an iterator that
also yields each partial result::
- itertools.accumulate([1,2,3,4,5]) =>
+ itertools.accumulate([1, 2, 3, 4, 5]) =>
1, 3, 6, 10, 15
- itertools.accumulate([1,2,3,4,5], operator.mul) =>
+ itertools.accumulate([1, 2, 3, 4, 5], operator.mul) =>
1, 2, 6, 24, 120
@@ -1155,7 +1156,7 @@ But it would be best of all if I had simply used a ``for`` loop::
Or the :func:`sum` built-in and a generator expression::
- total = sum(b for a,b in items)
+ total = sum(b for a, b in items)
Many uses of :func:`functools.reduce` are clearer when written as ``for`` loops.
diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst
index 54c3fe379ac33afb653fb11a27079ce9cc0cd613..50cde3595034b504d21ea5ca56bb5a22beac7442 100644
--- a/Doc/howto/instrumentation.rst
+++ b/Doc/howto/instrumentation.rst
@@ -316,6 +316,21 @@ Available static markers
Fires when the Python interpreter finishes a garbage collection
cycle. ``arg0`` is the number of collected objects.
+.. c:function:: import__find__load__start(str modulename)
+
+ Fires before :mod:`importlib` attempts to find and load the module.
+ ``arg0`` is the module name.
+
+ .. versionadded:: 3.7
+
+.. c:function:: import__find__load__done(str modulename, int found)
+
+ Fires after :mod:`importlib`'s find_and_load function is called.
+ ``arg0`` is the module name, ``arg1`` indicates if module was
+ successfully loaded.
+
+ .. versionadded:: 3.7
+
SystemTap Tapsets
-----------------
@@ -354,13 +369,13 @@ available:
.. c:function:: python.function.entry(str filename, str funcname, int lineno, frameptr)
This probe point indicates that execution of a Python function has begun.
- It is only triggered for pure-python (bytecode) functions.
+ It is only triggered for pure-Python (bytecode) functions.
.. c:function:: python.function.return(str filename, str funcname, int lineno, frameptr)
This probe point is the converse of :c:func:`python.function.return`, and
indicates that execution of a Python function has ended (either via
- ``return``, or via an exception). It is only triggered for pure-python
+ ``return``, or via an exception). It is only triggered for pure-Python
(bytecode) functions.
diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst
index 96d550c457b819975f98ab21a30737e65080c51b..b1930a791fca5b51cc0cc070a4c0ec156609a6fe 100644
--- a/Doc/howto/logging-cookbook.rst
+++ b/Doc/howto/logging-cookbook.rst
@@ -1069,7 +1069,7 @@ to indicate additional contextual information to be added to the log). So
you cannot directly make logging calls using :meth:`str.format` or
:class:`string.Template` syntax, because internally the logging package
uses %-formatting to merge the format string and the variable arguments.
-There would no changing this while preserving backward compatibility, since
+There would be no changing this while preserving backward compatibility, since
all logging calls which are out there in existing code will be using %-format
strings.
diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst
index 9914f8d154ab88af656e97a276fa9ea65550c1be..2a2282e9ecf068ef495fd8a2744b7ccfeea70e2b 100644
--- a/Doc/howto/logging.rst
+++ b/Doc/howto/logging.rst
@@ -326,7 +326,7 @@ favourite beverage and carry on.
If your logging needs are simple, then use the above examples to incorporate
logging into your own scripts, and if you run into problems or don't
understand something, please post a question on the comp.lang.python Usenet
-group (available at https://groups.google.com/group/comp.lang.python) and you
+group (available at https://groups.google.com/forum/#!forum/comp.lang.python) and you
should receive help before too long.
Still here? You can carry on reading the next few sections, which provide a
@@ -971,7 +971,7 @@ provided:
The :class:`NullHandler`, :class:`StreamHandler` and :class:`FileHandler`
classes are defined in the core logging package. The other handlers are
-defined in a sub- module, :mod:`logging.handlers`. (There is also another
+defined in a sub-module, :mod:`logging.handlers`. (There is also another
sub-module, :mod:`logging.config`, for configuration functionality.)
Logged messages are formatted for presentation through instances of the
diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst
index 88b01779d2a5f49a611c093f6550f486905b6448..3be6bb380d663b28040212c54dfe0269ce84e6a1 100644
--- a/Doc/howto/pyporting.rst
+++ b/Doc/howto/pyporting.rst
@@ -433,12 +433,12 @@ to make sure everything functions as expected in both versions of Python.
.. _Futurize: http://python-future.org/automatic_conversion.html
.. _importlib: https://docs.python.org/3/library/importlib.html#module-importlib
.. _importlib2: https://pypi.org/project/importlib2
-.. _Modernize: https://python-modernize.readthedocs.org/en/latest/
+.. _Modernize: https://python-modernize.readthedocs.io/
.. _mypy: http://mypy-lang.org/
.. _Porting to Python 3: http://python3porting.com/
.. _Pylint: https://pypi.org/project/pylint
-.. _Python 3 Q & A: https://ncoghlan-devs-python-notes.readthedocs.org/en/latest/python3/questions_and_answers.html
+.. _Python 3 Q & A: https://ncoghlan-devs-python-notes.readthedocs.io/en/latest/python3/questions_and_answers.html
.. _pytype: https://github.com/google/pytype
.. _python-future: http://python-future.org/
@@ -449,4 +449,4 @@ to make sure everything functions as expected in both versions of Python.
.. _"What's New": https://docs.python.org/3/whatsnew/index.html
-.. _Why Python 3 exists: http://www.snarky.ca/why-python-3-exists
+.. _Why Python 3 exists: https://snarky.ca/why-python-3-exists
diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst
index 65b09459357b5c7638f087345bb2d85fc1a400f7..b09f748a9227330fd89551f90c278654cc4574b3 100644
--- a/Doc/howto/regex.rst
+++ b/Doc/howto/regex.rst
@@ -371,7 +371,7 @@ for a complete listing.
| | returns them as an :term:`iterator`. |
+------------------+-----------------------------------------------+
-:meth:`~re.pattern.match` and :meth:`~re.pattern.search` return ``None`` if no match can be found. If
+:meth:`~re.Pattern.match` and :meth:`~re.Pattern.search` return ``None`` if no match can be found. If
they're successful, a :ref:`match object ` instance is returned,
containing information about the match: where it starts and ends, the substring
it matched, and more.
@@ -393,7 +393,7 @@ Python interpreter, import the :mod:`re` module, and compile a RE::
Now, you can try matching various strings against the RE ``[a-z]+``. An empty
string shouldn't match at all, since ``+`` means 'one or more repetitions'.
-:meth:`~re.pattern.match` should return ``None`` in this case, which will cause the
+:meth:`~re.Pattern.match` should return ``None`` in this case, which will cause the
interpreter to print no output. You can explicitly print the result of
:meth:`!match` to make this clear. ::
@@ -402,12 +402,12 @@ interpreter to print no output. You can explicitly print the result of
None
Now, let's try it on a string that it should match, such as ``tempo``. In this
-case, :meth:`~re.pattern.match` will return a :ref:`match object `, so you
+case, :meth:`~re.Pattern.match` will return a :ref:`match object `, so you
should store the result in a variable for later use. ::
>>> m = p.match('tempo')
>>> m
- <_sre.SRE_Match object; span=(0, 5), match='tempo'>
+
Now you can query the :ref:`match object ` for information
about the matching string. Match object instances
@@ -435,18 +435,18 @@ Trying these methods will soon clarify their meaning::
>>> m.span()
(0, 5)
-:meth:`~re.match.group` returns the substring that was matched by the RE. :meth:`~re.match.start`
-and :meth:`~re.match.end` return the starting and ending index of the match. :meth:`~re.match.span`
-returns both start and end indexes in a single tuple. Since the :meth:`~re.pattern.match`
+:meth:`~re.Match.group` returns the substring that was matched by the RE. :meth:`~re.Match.start`
+and :meth:`~re.Match.end` return the starting and ending index of the match. :meth:`~re.Match.span`
+returns both start and end indexes in a single tuple. Since the :meth:`~re.Pattern.match`
method only checks if the RE matches at the start of a string, :meth:`!start`
-will always be zero. However, the :meth:`~re.pattern.search` method of patterns
+will always be zero. However, the :meth:`~re.Pattern.search` method of patterns
scans through the string, so the match may not start at zero in that
case. ::
>>> print(p.match('::: message'))
None
>>> m = p.search('::: message'); print(m)
- <_sre.SRE_Match object; span=(4, 11), match='message'>
+
>>> m.group()
'message'
>>> m.span()
@@ -464,7 +464,7 @@ In actual programs, the most common style is to store the
print('No match')
Two pattern methods return all of the matches for a pattern.
-:meth:`~re.pattern.findall` returns a list of matching strings::
+:meth:`~re.Pattern.findall` returns a list of matching strings::
>>> p = re.compile(r'\d+')
>>> p.findall('12 drummers drumming, 11 pipers piping, 10 lords a-leaping')
@@ -504,7 +504,7 @@ the RE string added as the first argument, and still return either ``None`` or a
>>> print(re.match(r'From\s+', 'Fromage amk'))
None
>>> re.match(r'From\s+', 'From amk Thu May 14 19:12:10 1998') #doctest: +ELLIPSIS
- <_sre.SRE_Match object; span=(0, 5), match='From '>
+
Under the hood, these functions simply create a pattern object for you
and call the appropriate method on it. They also store the compiled
@@ -710,7 +710,7 @@ given location, they can obviously be matched an infinite number of times.
line, the RE to use is ``^From``. ::
>>> print(re.search('^From', 'From Here to Eternity')) #doctest: +ELLIPSIS
- <_sre.SRE_Match object; span=(0, 4), match='From'>
+
>>> print(re.search('^From', 'Reciting From Memory'))
None
@@ -721,11 +721,11 @@ given location, they can obviously be matched an infinite number of times.
or any location followed by a newline character. ::
>>> print(re.search('}$', '{block}')) #doctest: +ELLIPSIS
- <_sre.SRE_Match object; span=(6, 7), match='}'>
+
>>> print(re.search('}$', '{block} '))
None
>>> print(re.search('}$', '{block}\n')) #doctest: +ELLIPSIS
- <_sre.SRE_Match object; span=(6, 7), match='}'>
+
To match a literal ``'$'``, use ``\$`` or enclose it inside a character class,
as in ``[$]``.
@@ -750,7 +750,7 @@ given location, they can obviously be matched an infinite number of times.
>>> p = re.compile(r'\bclass\b')
>>> print(p.search('no class at all'))
- <_sre.SRE_Match object; span=(3, 8), match='class'>
+
>>> print(p.search('the declassified algorithm'))
None
>>> print(p.search('one subclass is'))
@@ -768,7 +768,7 @@ given location, they can obviously be matched an infinite number of times.
>>> print(p.search('no class at all'))
None
>>> print(p.search('\b' + 'class' + '\b'))
- <_sre.SRE_Match object; span=(0, 7), match='\x08class\x08'>
+
Second, inside a character class, where there's no use for this assertion,
``\b`` represents the backspace character, for compatibility with Python's
@@ -812,8 +812,8 @@ of a group with a repeating qualifier, such as ``*``, ``+``, ``?``, or
Groups indicated with ``'('``, ``')'`` also capture the starting and ending
index of the text that they match; this can be retrieved by passing an argument
-to :meth:`~re.match.group`, :meth:`~re.match.start`, :meth:`~re.match.end`, and
-:meth:`~re.match.span`. Groups are
+to :meth:`~re.Match.group`, :meth:`~re.Match.start`, :meth:`~re.Match.end`, and
+:meth:`~re.Match.span`. Groups are
numbered starting with 0. Group 0 is always present; it's the whole RE, so
:ref:`match object ` methods all have group 0 as their default
argument. Later we'll see how to express groups that don't capture the span
@@ -839,13 +839,13 @@ from left to right. ::
>>> m.group(2)
'b'
-:meth:`~re.match.group` can be passed multiple group numbers at a time, in which case it
+:meth:`~re.Match.group` can be passed multiple group numbers at a time, in which case it
will return a tuple containing the corresponding values for those groups. ::
>>> m.group(2,1,2)
('b', 'abc', 'b')
-The :meth:`~re.match.groups` method returns a tuple containing the strings for all the
+The :meth:`~re.Match.groups` method returns a tuple containing the strings for all the
subgroups, from 1 up to however many there are. ::
>>> m.groups()
@@ -1070,7 +1070,7 @@ using the following pattern methods:
Splitting Strings
-----------------
-The :meth:`~re.pattern.split` method of a pattern splits a string apart
+The :meth:`~re.Pattern.split` method of a pattern splits a string apart
wherever the RE matches, returning a list of the pieces. It's similar to the
:meth:`~str.split` method of strings but provides much more generality in the
delimiters that you can split by; string :meth:`!split` only supports splitting by
@@ -1125,7 +1125,7 @@ Search and Replace
------------------
Another common task is to find all the matches for a pattern, and replace them
-with a different string. The :meth:`~re.pattern.sub` method takes a replacement value,
+with a different string. The :meth:`~re.Pattern.sub` method takes a replacement value,
which can be either a string or a function, and the string to be processed.
.. method:: .sub(replacement, string[, count=0])
@@ -1139,7 +1139,7 @@ which can be either a string or a function, and the string to be processed.
replaced; *count* must be a non-negative integer. The default value of 0 means
to replace all occurrences.
-Here's a simple example of using the :meth:`~re.pattern.sub` method. It replaces colour
+Here's a simple example of using the :meth:`~re.Pattern.sub` method. It replaces colour
names with the word ``colour``::
>>> p = re.compile('(blue|white|red)')
@@ -1148,7 +1148,7 @@ names with the word ``colour``::
>>> p.sub('colour', 'blue socks and red shoes', count=1)
'colour socks and red shoes'
-The :meth:`~re.pattern.subn` method does the same work, but returns a 2-tuple containing the
+The :meth:`~re.Pattern.subn` method does the same work, but returns a 2-tuple containing the
new string value and the number of replacements that were performed::
>>> p = re.compile('(blue|white|red)')
@@ -1157,12 +1157,12 @@ new string value and the number of replacements that were performed::
>>> p.subn('colour', 'no colours at all')
('no colours at all', 0)
-Empty matches are replaced only when they're not adjacent to a previous match.
+Empty matches are replaced only when they're not adjacent to a previous empty match.
::
>>> p = re.compile('x*')
>>> p.sub('-', 'abxd')
- '-a-b-d-'
+ '-a-b--d-'
If *replacement* is a string, any backslash escapes in it are processed. That
is, ``\n`` is converted to a single newline character, ``\r`` is converted to a
diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst
index ec5d48e64314845dffbbacd410b3ac0bb51bdbf6..be1fefb35a71f0970d5e84aadb93f011d79f73f9 100644
--- a/Doc/howto/unicode.rst
+++ b/Doc/howto/unicode.rst
@@ -216,10 +216,10 @@ difficult reading. `A chronology `_ of the
origin and development of Unicode is also available on the site.
To help understand the standard, Jukka Korpela has written `an introductory
-guide `_ to reading the
+guide `_ to reading the
Unicode character tables.
-Another `good introductory article `_
+Another `good introductory article `_
was written by Joel Spolsky.
If this introduction didn't make things clear to you, you should try
reading this alternate article before continuing.
@@ -489,7 +489,7 @@ References
Some good alternative discussions of Python's Unicode support are:
* `Processing Text Files in Python 3 `_, by Nick Coghlan.
-* `Pragmatic Unicode `_, a PyCon 2012 presentation by Ned Batchelder.
+* `Pragmatic Unicode `_, a PyCon 2012 presentation by Ned Batchelder.
The :class:`str` type is described in the Python library reference at
:ref:`textseq`.
diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst
index 7505e7564422abc34a0467794c5afc14222bfaa2..046a88af62f0b3e31ddb2ce9cfc69642cc9cf366 100644
--- a/Doc/howto/urllib2.rst
+++ b/Doc/howto/urllib2.rst
@@ -411,7 +411,7 @@ fetched, particularly the headers sent by the server. It is currently an
:class:`http.client.HTTPMessage` instance.
Typical headers include 'Content-length', 'Content-type', and so on. See the
-`Quick Reference to HTTP Headers `_
+`Quick Reference to HTTP Headers `_
for a useful listing of HTTP headers with brief explanations of their meaning
and use.
diff --git a/Doc/includes/run-func.c b/Doc/includes/run-func.c
index ead7bdd23209a3d45eb498731454f427bf4fcb42..9caf1fdb20104af60af4e275acaa5682e180055f 100644
--- a/Doc/includes/run-func.c
+++ b/Doc/includes/run-func.c
@@ -3,7 +3,7 @@
int
main(int argc, char *argv[])
{
- PyObject *pName, *pModule, *pDict, *pFunc;
+ PyObject *pName, *pModule, *pFunc;
PyObject *pArgs, *pValue;
int i;
diff --git a/Doc/includes/tzinfo-examples.py b/Doc/includes/tzinfo_examples.py
similarity index 98%
rename from Doc/includes/tzinfo-examples.py
rename to Doc/includes/tzinfo_examples.py
index ae5a5092665f09fdb4e002b6efb66de1961cf9ea..9b9e32a553e7d8e82037e9383e22c856a063328e 100644
--- a/Doc/includes/tzinfo-examples.py
+++ b/Doc/includes/tzinfo_examples.py
@@ -1,4 +1,4 @@
-from datetime import tzinfo, timedelta, datetime, timezone
+from datetime import tzinfo, timedelta, datetime
ZERO = timedelta(0)
HOUR = timedelta(hours=1)
diff --git a/Doc/install/index.rst b/Doc/install/index.rst
index 484231323896f04e2a7883ea95a6d0d6b69b47fa..f6a8cd6833a9ad18908b6cdca1b17de7d3440563 100644
--- a/Doc/install/index.rst
+++ b/Doc/install/index.rst
@@ -36,7 +36,7 @@ modules and extensions.
This guide only covers the basic tools for building and distributing
extensions that are provided as part of this version of Python. Third party
tools offer easier to use and more secure alternatives. Refer to the `quick
- recommendations section `__
+ recommendations section `__
in the Python Packaging User Guide for more information.
@@ -554,10 +554,10 @@ C headers ``--install-headers``
These override options can be relative, absolute,
or explicitly defined in terms of one of the installation base directories.
-(There are two installation base directories, and they are normally the same---
-they only differ when you use the Unix "prefix scheme" and supply different
-``--prefix`` and ``--exec-prefix`` options; using ``--install-lib`` will
-override values computed or given for ``--install-purelib`` and
+(There are two installation base directories, and they are normally the
+same---they only differ when you use the Unix "prefix scheme" and supply
+different ``--prefix`` and ``--exec-prefix`` options; using ``--install-lib``
+will override values computed or given for ``--install-purelib`` and
``--install-platlib``, and is recommended for schemes that don't make a
difference between Python and extension modules.)
@@ -584,10 +584,10 @@ in this case.)
If you maintain Python on Windows, you might want third-party modules to live in
a subdirectory of :file:`{prefix}`, rather than right in :file:`{prefix}`
-itself. This is almost as easy as customizing the script installation directory
----you just have to remember that there are two types of modules to worry about,
-Python and extension modules, which can conveniently be both controlled by one
-option::
+itself. This is almost as easy as customizing the script installation
+directory---you just have to remember that there are two types of modules
+to worry about, Python and extension modules, which can conveniently be both
+controlled by one option::
python setup.py install --install-lib=Site
diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst
index 18f5b29396c9c26e05a300ccda2aa64c8a427be1..8af828bb8a105f35c3043e06d76f1c083835dacc 100644
--- a/Doc/installing/index.rst
+++ b/Doc/installing/index.rst
@@ -48,7 +48,7 @@ Key terms
repository of open source licensed packages made available for use by
other Python users.
* the `Python Packaging Authority
- `__ are the group of
+ `__ is the group of
developers and documentation authors responsible for the maintenance and
evolution of the standard packaging tools and the associated metadata and
file format standards. They maintain a variety of tools, documentation,
diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst
index 1ab05a62ad458fbbf38a56fd11e1fbaf0b443474..fa4b0a9645531c4792d0116a226cd9ea13b8f83a 100644
--- a/Doc/library/2to3.rst
+++ b/Doc/library/2to3.rst
@@ -345,20 +345,20 @@ and off individually. They are described here in more detail.
Converts calls to various functions in the :mod:`operator` module to other,
but equivalent, function calls. When needed, the appropriate ``import``
- statements are added, e.g. ``import collections``. The following mapping
+ statements are added, e.g. ``import collections.abc``. The following mapping
are made:
- ================================== ==========================================
+ ================================== =============================================
From To
- ================================== ==========================================
- ``operator.isCallable(obj)`` ``hasattr(obj, '__call__')``
+ ================================== =============================================
+ ``operator.isCallable(obj)`` ``callable(obj)``
``operator.sequenceIncludes(obj)`` ``operator.contains(obj)``
- ``operator.isSequenceType(obj)`` ``isinstance(obj, collections.Sequence)``
- ``operator.isMappingType(obj)`` ``isinstance(obj, collections.Mapping)``
+ ``operator.isSequenceType(obj)`` ``isinstance(obj, collections.abc.Sequence)``
+ ``operator.isMappingType(obj)`` ``isinstance(obj, collections.abc.Mapping)``
``operator.isNumberType(obj)`` ``isinstance(obj, numbers.Number)``
``operator.repeat(obj, n)`` ``operator.mul(obj, n)``
``operator.irepeat(obj, n)`` ``operator.imul(obj, n)``
- ================================== ==========================================
+ ================================== =============================================
.. 2to3fixer:: paren
@@ -385,7 +385,7 @@ and off individually. They are described here in more detail.
.. 2to3fixer:: reload
- Converts :func:`reload` to :func:`imp.reload`.
+ Converts :func:`reload` to :func:`importlib.reload`.
.. 2to3fixer:: renames
diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst
index 73d8b6b7e8aff025e6cc552fa7dcdc5ea39a744f..e3d749e60178478fe8645fa9b6ad5465b449ef8f 100644
--- a/Doc/library/__future__.rst
+++ b/Doc/library/__future__.rst
@@ -90,6 +90,11 @@ language using this mechanism:
| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: |
| | | | *StopIteration handling inside generators* |
+------------------+-------------+--------------+---------------------------------------------+
+| annotations | 3.7.0b1 | 4.0 | :pep:`563`: |
+| | | | *Postponed evaluation of annotations* |
++------------------+-------------+--------------+---------------------------------------------+
+
+.. XXX Adding a new entry? Remember to update simple_stmts.rst, too.
.. seealso::
diff --git a/Doc/library/_dummy_thread.rst b/Doc/library/_dummy_thread.rst
index ebce74d5a2252f77b6c3f758919c177561acf5c3..7dccbc55475aae73c684418af76b35e1e22b7d97 100644
--- a/Doc/library/_dummy_thread.rst
+++ b/Doc/library/_dummy_thread.rst
@@ -6,18 +6,15 @@
**Source code:** :source:`Lib/_dummy_thread.py`
---------------
-
-This module provides a duplicate interface to the :mod:`_thread` module. It is
-meant to be imported when the :mod:`_thread` module is not provided on a
-platform.
+.. deprecated:: 3.7
+ Python now always has threading enabled. Please use :mod:`_thread`
+ (or, better, :mod:`threading`) instead.
-Suggested usage is::
+--------------
- try:
- import _thread
- except ImportError:
- import _dummy_thread as _thread
+This module provides a duplicate interface to the :mod:`_thread` module.
+It was meant to be imported when the :mod:`_thread` module was not provided
+on a platform.
Be careful to not use this module where deadlock might occur from a thread being
created that blocks waiting for another thread to be created. This often occurs
diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst
index 0d2d818f5f90b00cf1e4779a61972fe066a83f53..acffabf24bad5fcf1c5570e475eff1cdfd71343c 100644
--- a/Doc/library/_thread.rst
+++ b/Doc/library/_thread.rst
@@ -23,14 +23,10 @@ threading API built on top of this module.
single: pthreads
pair: threads; POSIX
-The module is optional. It is supported on Windows, Linux, SGI IRIX, Solaris
-2.x, as well as on systems that have a POSIX thread (a.k.a. "pthread")
-implementation. For systems lacking the :mod:`_thread` module, the
-:mod:`_dummy_thread` module is available. It duplicates this module's interface
-and can be used as a drop-in replacement.
-
-It defines the following constants and functions:
+.. versionchanged:: 3.7
+ This module used to be optional, it is now always available.
+This module defines the following constants and functions:
.. exception:: error
@@ -105,7 +101,8 @@ It defines the following constants and functions:
memory page size - platform documentation should be referred to for more
information (4 KiB pages are common; using multiples of 4096 for the stack size is
the suggested approach in the absence of more specific information).
- Availability: Windows, systems with POSIX threads.
+
+ .. availability:: Windows, systems with POSIX threads.
.. data:: TIMEOUT_MAX
diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst
index 70710761a39555b9151e194439747818f3ceb529..fc39a2e7582b3a8dc95aff0b7b2a1ec3a0f1a984 100644
--- a/Doc/library/abc.rst
+++ b/Doc/library/abc.rst
@@ -18,10 +18,10 @@ see the PEP for why this was added to Python. (See also :pep:`3141` and the
:mod:`numbers` module regarding a type hierarchy for numbers based on ABCs.)
The :mod:`collections` module has some concrete classes that derive from
-ABCs; these can, of course, be further derived. In addition the
+ABCs; these can, of course, be further derived. In addition, the
:mod:`collections.abc` submodule has some ABCs that can be used to test whether
-a class or instance provides a particular interface, for example, is it
-hashable or a mapping.
+a class or instance provides a particular interface, for example, if it is
+hashable or if it is a mapping.
This module provides the metaclass :class:`ABCMeta` for defining ABCs and
@@ -217,7 +217,7 @@ The :mod:`abc` module also provides the following decorator:
the descriptor must identify itself as abstract using
:attr:`__isabstractmethod__`. In general, this attribute should be ``True``
if any of the methods used to compose the descriptor are abstract. For
- example, Python's built-in property does the equivalent of::
+ example, Python's built-in :class:`property` does the equivalent of::
class Descriptor:
...
diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index a9cb5c5357b60742dfe3aed6e65aab39b69d5b7b..cef197f30555818d539db7b3d587251f640bf4ac 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -844,6 +844,8 @@ values are:
Note that ``nargs=1`` produces a list of one item. This is different from
the default, in which the item is produced by itself.
+.. index:: single: ? (question mark); in argparse module
+
* ``'?'``. One argument will be consumed from the command line if possible, and
produced as a single item. If no command-line argument is present, the value from
default_ will be produced. Note that for optional arguments, there is an
@@ -876,6 +878,8 @@ values are:
Namespace(infile=<_io.TextIOWrapper name='' encoding='UTF-8'>,
outfile=<_io.TextIOWrapper name='' encoding='UTF-8'>)
+.. index:: single: * (asterisk); in argparse module
+
* ``'*'``. All command-line arguments present are gathered into a list. Note that
it generally doesn't make much sense to have more than one positional argument
with ``nargs='*'``, but multiple optional arguments with ``nargs='*'`` is
@@ -888,6 +892,8 @@ values are:
>>> parser.parse_args('a b --foo x y --bar 1 2'.split())
Namespace(bar=['1', '2'], baz=['a', 'b'], foo=['x', 'y'])
+.. index:: single: + (plus); in argparse module
+
* ``'+'``. Just like ``'*'``, all command-line args present are gathered into a
list. Additionally, an error message will be generated if there wasn't at
least one command-line argument present. For example::
@@ -1539,8 +1545,8 @@ Sub-commands
.. method:: ArgumentParser.add_subparsers([title], [description], [prog], \
[parser_class], [action], \
- [option_string], [dest], [help], \
- [metavar])
+ [option_string], [dest], [required], \
+ [help], [metavar])
Many programs split up their functionality into a number of sub-commands,
for example, the ``svn`` program can invoke sub-commands like ``svn
@@ -1576,6 +1582,9 @@ Sub-commands
* dest_ - name of the attribute under which sub-command name will be
stored; by default ``None`` and no value is stored
+ * required_ - Whether or not a subcommand must be provided, by default
+ ``False``.
+
* help_ - help for sub-parser group in help output, by default ``None``
* metavar_ - string presenting available sub-commands in help; by default it
@@ -1987,6 +1996,45 @@ Exiting methods
This method prints a usage message including the *message* to the
standard error and terminates the program with a status code of 2.
+
+Intermixed parsing
+^^^^^^^^^^^^^^^^^^
+
+.. method:: ArgumentParser.parse_intermixed_args(args=None, namespace=None)
+.. method:: ArgumentParser.parse_known_intermixed_args(args=None, namespace=None)
+
+A number of Unix commands allow the user to intermix optional arguments with
+positional arguments. The :meth:`~ArgumentParser.parse_intermixed_args`
+and :meth:`~ArgumentParser.parse_known_intermixed_args` methods
+support this parsing style.
+
+These parsers do not support all the argparse features, and will raise
+exceptions if unsupported features are used. In particular, subparsers,
+``argparse.REMAINDER``, and mutually exclusive groups that include both
+optionals and positionals are not supported.
+
+The following example shows the difference between
+:meth:`~ArgumentParser.parse_known_args` and
+:meth:`~ArgumentParser.parse_intermixed_args`: the former returns ``['2',
+'3']`` as unparsed arguments, while the latter collects all the positionals
+into ``rest``. ::
+
+ >>> parser = argparse.ArgumentParser()
+ >>> parser.add_argument('--foo')
+ >>> parser.add_argument('cmd')
+ >>> parser.add_argument('rest', nargs='*', type=int)
+ >>> parser.parse_known_args('doit 1 --foo bar 2 3'.split())
+ (Namespace(cmd='doit', foo='bar', rest=[1]), ['2', '3'])
+ >>> parser.parse_intermixed_args('doit 1 --foo bar 2 3'.split())
+ Namespace(cmd='doit', foo='bar', rest=[1, 2, 3])
+
+:meth:`~ArgumentParser.parse_known_intermixed_args` returns a two item tuple
+containing the populated namespace and the list of remaining argument strings.
+:meth:`~ArgumentParser.parse_intermixed_args` raises an error if there are any
+remaining unparsed argument strings.
+
+.. versionadded:: 3.7
+
.. _upgrading-optparse-code:
Upgrading optparse code
@@ -2020,9 +2068,8 @@ A partial upgrade path from :mod:`optparse` to :mod:`argparse`:
called ``options``, now in the :mod:`argparse` context is called ``args``.
* Replace :meth:`optparse.OptionParser.disable_interspersed_args`
- by setting ``nargs`` of a positional argument to `argparse.REMAINDER`_, or
- use :meth:`~ArgumentParser.parse_known_args` to collect unparsed argument
- strings in a separate list.
+ by using :meth:`~ArgumentParser.parse_intermixed_args` instead of
+ :meth:`~ArgumentParser.parse_args`.
* Replace callback actions and the ``callback_*`` keyword arguments with
``type`` or ``action`` arguments.
diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst
index 6376f5fe441011b2cec384717bab641f1d771f75..3c73f748acbe36549290abd04264cd4e4a18c922 100644
--- a/Doc/library/ast.rst
+++ b/Doc/library/ast.rst
@@ -41,6 +41,9 @@ Node classes
with alternatives (aka "sums"), the left-hand side class is abstract: only
instances of specific constructor nodes are ever created.
+ .. index:: single: ? (question mark); in AST grammar
+ .. index:: single: * (asterisk); in AST grammar
+
.. attribute:: _fields
Each concrete class has an attribute :attr:`_fields` which gives the names
@@ -143,9 +146,13 @@ and classes for traversing abstract syntax trees:
.. function:: get_docstring(node, clean=True)
Return the docstring of the given *node* (which must be a
- :class:`FunctionDef`, :class:`ClassDef` or :class:`Module` node), or ``None``
- if it has no docstring. If *clean* is true, clean up the docstring's
- indentation with :func:`inspect.cleandoc`.
+ :class:`FunctionDef`, :class:`AsyncFunctionDef`, :class:`ClassDef`,
+ or :class:`Module` node), or ``None`` if it has no docstring.
+ If *clean* is true, clean up the docstring's indentation with
+ :func:`inspect.cleandoc`.
+
+ .. versionchanged:: 3.5
+ :class:`AsyncFunctionDef` is now supported.
.. function:: fix_missing_locations(node)
@@ -263,5 +270,5 @@ and classes for traversing abstract syntax trees:
.. seealso::
- `Green Tree Snakes `_, an external documentation resource, has good
+ `Green Tree Snakes `_, an external documentation resource, has good
details on working with Python ASTs.
diff --git a/Doc/library/asyncio-api-index.rst b/Doc/library/asyncio-api-index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..d5b5659abc65e2fa4e1ec513996ebcdb20042e49
--- /dev/null
+++ b/Doc/library/asyncio-api-index.rst
@@ -0,0 +1,218 @@
+.. currentmodule:: asyncio
+
+
+====================
+High-level API Index
+====================
+
+This page lists all high-level async/await enabled asyncio APIs.
+
+
+Tasks
+=====
+
+Utilities to run asyncio programs, create Tasks, and
+await on multiple things with timeouts.
+
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - :func:`run`
+ - Create event loop, run a coroutine, close the loop.
+
+ * - :func:`create_task`
+ - Start an asyncio Task.
+
+ * - ``await`` :func:`sleep`
+ - Sleep for a number of seconds.
+
+ * - ``await`` :func:`gather`
+ - Schedule and wait for things concurrently.
+
+ * - ``await`` :func:`wait_for`
+ - Run with a timeout.
+
+ * - ``await`` :func:`shield`
+ - Shield from cancellation.
+
+ * - ``await`` :func:`wait`
+ - Monitor for completion.
+
+ * - :func:`current_task`
+ - Return the current Task.
+
+ * - :func:`all_tasks`
+ - Return all tasks for an event loop.
+
+ * - :class:`Task`
+ - Task object.
+
+ * - :func:`run_coroutine_threadsafe`
+ - Schedule a coroutine from another OS thread.
+
+ * - ``for in`` :func:`as_completed`
+ - Monitor for completion with a ``for`` loop.
+
+
+.. rubric:: Examples
+
+* :ref:`Using asyncio.gather() to run things in parallel
+ `.
+
+* :ref:`Using asyncio.wait_for() to enforce a timeout
+ `.
+
+* :ref:`Cancellation `.
+
+* :ref:`Using asyncio.sleep() `.
+
+* See also the main :ref:`Tasks documentation page `.
+
+
+Queues
+======
+
+Queues should be used to distribute work amongst multiple asyncio Tasks,
+implement connection pools, and pub/sub patterns.
+
+
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - :class:`Queue`
+ - A FIFO queue.
+
+ * - :class:`PriorityQueue`
+ - A priority queue.
+
+ * - :class:`LifoQueue`
+ - A LIFO queue.
+
+
+.. rubric:: Examples
+
+* :ref:`Using asyncio.Queue to distribute workload between several
+ Tasks `.
+
+* See also the :ref:`Queues documentation page `.
+
+
+Subprocesses
+============
+
+Utilities to spawn subprocesses and run shell commands.
+
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - ``await`` :func:`create_subprocess_exec`
+ - Create a subprocess.
+
+ * - ``await`` :func:`create_subprocess_shell`
+ - Run a shell command.
+
+
+.. rubric:: Examples
+
+* :ref:`Executing a shell command `.
+
+* See also the :ref:`subprocess APIs `
+ documentation.
+
+
+Streams
+=======
+
+High-level APIs to work with network IO.
+
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - ``await`` :func:`open_connection`
+ - Establish a TCP connection.
+
+ * - ``await`` :func:`open_unix_connection`
+ - Establish a Unix socket connection.
+
+ * - ``await`` :func:`start_server`
+ - Start a TCP server.
+
+ * - ``await`` :func:`start_unix_server`
+ - Start a Unix socket server.
+
+ * - :class:`StreamReader`
+ - High-level async/await object to receive network data.
+
+ * - :class:`StreamWriter`
+ - High-level async/await object to send network data.
+
+
+.. rubric:: Examples
+
+* :ref:`Example TCP client `.
+
+* See also the :ref:`streams APIs `
+ documentation.
+
+
+Synchronization
+===============
+
+Threading-like synchronization primitives that can be used in Tasks.
+
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - :class:`Lock`
+ - A mutex lock.
+
+ * - :class:`Event`
+ - An event object.
+
+ * - :class:`Condition`
+ - A condition object.
+
+ * - :class:`Semaphore`
+ - A semaphore.
+
+ * - :class:`BoundedSemaphore`
+ - A bounded semaphore.
+
+
+.. rubric:: Examples
+
+* :ref:`Using asyncio.Event `.
+
+* See also the documentation of asyncio
+ :ref:`synchronization primitives `.
+
+
+Exceptions
+==========
+
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+
+ * - :exc:`asyncio.TimeoutError`
+ - Raised on timeout by functions like :func:`wait_for`.
+ Keep in mind that ``asyncio.TimeoutError`` is **unrelated**
+ to the built-in :exc:`TimeoutError` exception.
+
+ * - :exc:`asyncio.CancelledError`
+ - Raised when a Task is cancelled. See also :meth:`Task.cancel`.
+
+
+.. rubric:: Examples
+
+* :ref:`Handling CancelledError to run code on cancellation request
+ `.
+
+* See also the full list of
+ :ref:`asyncio-specific exceptions `.
diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst
index e2ad3eb0196c93cb84ad59f5ceac563c9045f97d..b728803619297968d9c57fb4f2b8a5e34566de2e 100644
--- a/Doc/library/asyncio-dev.rst
+++ b/Doc/library/asyncio-dev.rst
@@ -2,417 +2,236 @@
.. _asyncio-dev:
-Develop with asyncio
-====================
+=======================
+Developing with asyncio
+=======================
-Asynchronous programming is different than classical "sequential" programming.
-This page lists common traps and explains how to avoid them.
+Asynchronous programming is different from classic "sequential"
+programming.
+This page lists common mistakes and traps and explains how
+to avoid them.
-.. _asyncio-debug-mode:
-Debug mode of asyncio
----------------------
+.. _asyncio-debug-mode:
-The implementation of :mod:`asyncio` has been written for performance.
-In order to ease the development of asynchronous code, you may wish to
-enable *debug mode*.
+Debug Mode
+==========
-To enable all debug checks for an application:
+By default asyncio runs in production mode. In order to ease
+the development asyncio has a *debug mode*.
-* Enable the asyncio debug mode globally by setting the environment variable
- :envvar:`PYTHONASYNCIODEBUG` to ``1``, or by calling :meth:`AbstractEventLoop.set_debug`.
-* Set the log level of the :ref:`asyncio logger ` to
- :py:data:`logging.DEBUG`. For example, call
- ``logging.basicConfig(level=logging.DEBUG)`` at startup.
-* Configure the :mod:`warnings` module to display :exc:`ResourceWarning`
- warnings. For example, use the ``-Wdefault`` command line option of Python to
- display them.
+There are several ways to enable asyncio debug mode:
-Examples debug checks:
+* Setting the :envvar:`PYTHONASYNCIODEBUG` environment variable to ``1``.
-* Log :ref:`coroutines defined but never "yielded from"
- `
-* :meth:`~AbstractEventLoop.call_soon` and :meth:`~AbstractEventLoop.call_at` methods
- raise an exception if they are called from the wrong thread.
-* Log the execution time of the selector
-* Log callbacks taking more than 100 ms to be executed. The
- :attr:`AbstractEventLoop.slow_callback_duration` attribute is the minimum
- duration in seconds of "slow" callbacks.
-* :exc:`ResourceWarning` warnings are emitted when transports and event loops
- are :ref:`not closed explicitly `.
+* Using the :option:`-X` ``dev`` Python command line option.
-.. seealso::
+* Passing ``debug=True`` to :func:`asyncio.run`.
- The :meth:`AbstractEventLoop.set_debug` method and the :ref:`asyncio logger
- `.
+* Calling :meth:`loop.set_debug`.
+In addition to enabling the debug mode, consider also:
-Cancellation
-------------
+* setting the log level of the :ref:`asyncio logger ` to
+ :py:data:`logging.DEBUG`, for example the following snippet of code
+ can be run at startup of the application::
-Cancellation of tasks is not common in classic programming. In asynchronous
-programming, not only is it something common, but you have to prepare your
-code to handle it.
+ logging.basicConfig(level=logging.DEBUG)
-Futures and tasks can be cancelled explicitly with their :meth:`Future.cancel`
-method. The :func:`wait_for` function cancels the waited task when the timeout
-occurs. There are many other cases where a task can be cancelled indirectly.
+* configuring the :mod:`warnings` module to display
+ :exc:`ResourceWarning` warnings. One way of doing that is by
+ using the :option:`-W` ``default`` command line option.
-Don't call :meth:`~Future.set_result` or :meth:`~Future.set_exception` method
-of :class:`Future` if the future is cancelled: it would fail with an exception.
-For example, write::
- if not fut.cancelled():
- fut.set_result('done')
+When the debug mode is enabled:
-Don't schedule directly a call to the :meth:`~Future.set_result` or the
-:meth:`~Future.set_exception` method of a future with
-:meth:`AbstractEventLoop.call_soon`: the future can be cancelled before its method
-is called.
+* asyncio checks for :ref:`coroutines that were not awaited
+ ` and logs them; this mitigates
+ the "forgotten await" pitfall.
-If you wait for a future, you should check early if the future was cancelled to
-avoid useless operations. Example::
+* Many non-threadsafe asyncio APIs (such as :meth:`loop.call_soon` and
+ :meth:`loop.call_at` methods) raise an exception if they are called
+ from a wrong thread.
- @coroutine
- def slow_operation(fut):
- if fut.cancelled():
- return
- # ... slow computation ...
- yield from fut
- # ...
+* The execution time of the I/O selector is logged if it takes too long to
+ perform an I/O operation.
-The :func:`shield` function can also be used to ignore cancellation.
+* Callbacks taking longer than 100ms are logged. The
+ :attr:`loop.slow_callback_duration` attribute can be used to set the
+ minimum execution duration in seconds that is considered "slow".
.. _asyncio-multithreading:
-Concurrency and multithreading
-------------------------------
+Concurrency and Multithreading
+==============================
-An event loop runs in a thread and executes all callbacks and tasks in the same
-thread. While a task is running in the event loop, no other task is running in
-the same thread. But when the task uses ``yield from``, the task is suspended
-and the event loop executes the next task.
+An event loop runs in a thread (typically the main thread) and executes
+all callbacks and Tasks in its thread. While a Task is running in the
+event loop, no other Tasks can run in the same thread. When a Task
+executes an ``await`` expression, the running Task gets suspended, and
+the event loop executes the next Task.
-To schedule a callback from a different thread, the
-:meth:`AbstractEventLoop.call_soon_threadsafe` method should be used. Example::
+To schedule a callback from a different OS thread, the
+:meth:`loop.call_soon_threadsafe` method should be used. Example::
loop.call_soon_threadsafe(callback, *args)
-Most asyncio objects are not thread safe. You should only worry if you access
-objects outside the event loop. For example, to cancel a future, don't call
-directly its :meth:`Future.cancel` method, but::
+Almost all asyncio objects are not thread safe, which is typically
+not a problem unless there is code that works with them from outside
+of a Task or a callback. If there's a need for such code to call a
+low-level asyncio API, the :meth:`loop.call_soon_threadsafe` method
+should be used, e.g.::
loop.call_soon_threadsafe(fut.cancel)
-To handle signals and to execute subprocesses, the event loop must be run in
-the main thread.
-
-To schedule a coroutine object from a different thread, the
+To schedule a coroutine object from a different OS thread, the
:func:`run_coroutine_threadsafe` function should be used. It returns a
:class:`concurrent.futures.Future` to access the result::
- future = asyncio.run_coroutine_threadsafe(coro_func(), loop)
- result = future.result(timeout) # Wait for the result with a timeout
-
-The :meth:`AbstractEventLoop.run_in_executor` method can be used with a thread pool
-executor to execute a callback in different thread to not block the thread of
-the event loop.
+ async def coro_func():
+ return await asyncio.sleep(1, 42)
-.. seealso::
+ # Later in another OS thread:
- The :ref:`Synchronization primitives ` section describes ways
- to synchronize tasks.
-
- The :ref:`Subprocess and threads ` section lists
- asyncio limitations to run subprocesses from different threads.
+ future = asyncio.run_coroutine_threadsafe(coro_func(), loop)
+ # Wait for the result:
+ result = future.result()
+To handle signals and to execute subprocesses, the event loop must be
+run in the main thread.
+The :meth:`loop.run_in_executor` method can be used with a
+:class:`concurrent.futures.ThreadPoolExecutor` to execute
+blocking code in a different OS thread without blocking the OS thread
+that the event loop runs in.
.. _asyncio-handle-blocking:
-Handle blocking functions correctly
------------------------------------
-
-Blocking functions should not be called directly. For example, if a function
-blocks for 1 second, other tasks are delayed by 1 second which can have an
-important impact on reactivity.
-
-For networking and subprocesses, the :mod:`asyncio` module provides high-level
-APIs like :ref:`protocols `.
+Running Blocking Code
+=====================
-An executor can be used to run a task in a different thread or even in a
-different process, to not block the thread of the event loop. See the
-:meth:`AbstractEventLoop.run_in_executor` method.
+Blocking (CPU-bound) code should not be called directly. For example,
+if a function performs a CPU-intensive calculation for 1 second,
+all concurrent asyncio Tasks and IO operations would be delayed
+by 1 second.
-.. seealso::
-
- The :ref:`Delayed calls ` section details how the
- event loop handles time.
+An executor can be used to run a task in a different thread or even in
+a different process to avoid blocking block the OS thread with the
+event loop. See the :meth:`loop.run_in_executor` method for more
+details.
.. _asyncio-logger:
Logging
--------
-
-The :mod:`asyncio` module logs information with the :mod:`logging` module in
-the logger ``'asyncio'``.
+=======
-The default log level for the :mod:`asyncio` module is :py:data:`logging.INFO`.
-For those not wanting such verbosity from :mod:`asyncio` the log level can
-be changed. For example, to change the level to :py:data:`logging.WARNING`:
+asyncio uses the :mod:`logging` module and all logging is performed
+via the ``"asyncio"`` logger.
-.. code-block:: none
+The default log level is :py:data:`logging.INFO`, which can be easily
+adjusted::
- logging.getLogger('asyncio').setLevel(logging.WARNING)
+ logging.getLogger("asyncio").setLevel(logging.WARNING)
.. _asyncio-coroutine-not-scheduled:
-Detect coroutine objects never scheduled
-----------------------------------------
-
-When a coroutine function is called and its result is not passed to
-:func:`ensure_future` or to the :meth:`AbstractEventLoop.create_task` method,
-the execution of the coroutine object will never be scheduled which is
-probably a bug. :ref:`Enable the debug mode of asyncio `
-to :ref:`log a warning ` to detect it.
+Detect never-awaited coroutines
+===============================
-Example with the bug::
+When a coroutine function is called, but not awaited
+(e.g. ``coro()`` instead of ``await coro()``)
+or the coroutine is not scheduled with :meth:`asyncio.create_task`, asyncio
+will emit a :exc:`RuntimeWarning`::
import asyncio
- @asyncio.coroutine
- def test():
+ async def test():
print("never scheduled")
+ async def main():
+ test()
+
+ asyncio.run(main())
+
+Output::
+
+ test.py:7: RuntimeWarning: coroutine 'test' was never awaited
test()
Output in debug mode::
- Coroutine test() at test.py:3 was never yielded from
- Coroutine object created at (most recent call last):
- File "test.py", line 7, in
- test()
+ test.py:7: RuntimeWarning: coroutine 'test' was never awaited
+ Coroutine created at (most recent call last)
+ File "../t.py", line 9, in
+ asyncio.run(main(), debug=True)
-The fix is to call the :func:`ensure_future` function or the
-:meth:`AbstractEventLoop.create_task` method with the coroutine object.
+ < .. >
-.. seealso::
+ File "../t.py", line 7, in main
+ test()
+ test()
+
+The usual fix is to either await the coroutine or call the
+:meth:`asyncio.create_task` function::
- :ref:`Pending task destroyed `.
+ async def main():
+ await test()
-Detect exceptions never consumed
---------------------------------
+Detect never-retrieved exceptions
+=================================
-Python usually calls :func:`sys.excepthook` on unhandled exceptions. If
-:meth:`Future.set_exception` is called, but the exception is never consumed,
-:func:`sys.excepthook` is not called. Instead, :ref:`a log is emitted
-` when the future is deleted by the garbage collector, with the
-traceback where the exception was raised.
+If a :meth:`Future.set_exception` is called but the Future object is
+never awaited on, the exception would never be propagated to the
+user code. In this case, asyncio would emit a log message when the
+Future object is garbage collected.
-Example of unhandled exception::
+Example of an unhandled exception::
import asyncio
- @asyncio.coroutine
- def bug():
+ async def bug():
raise Exception("not consumed")
- loop = asyncio.get_event_loop()
- asyncio.ensure_future(bug())
- loop.run_forever()
- loop.close()
+ async def main():
+ asyncio.create_task(bug())
+
+ asyncio.run(main())
Output::
Task exception was never retrieved
- future: exception=Exception('not consumed',)>
- Traceback (most recent call last):
- File "asyncio/tasks.py", line 237, in _step
- result = next(coro)
- File "asyncio/coroutines.py", line 141, in coro
- res = func(*args, **kw)
- File "test.py", line 5, in bug
- raise Exception("not consumed")
- Exception: not consumed
-
-:ref:`Enable the debug mode of asyncio ` to get the
-traceback where the task was created. Output in debug mode::
+ future:
+ exception=Exception('not consumed')>
- Task exception was never retrieved
- future: exception=Exception('not consumed',) created at test.py:8>
- source_traceback: Object created at (most recent call last):
- File "test.py", line 8, in
- asyncio.ensure_future(bug())
Traceback (most recent call last):
- File "asyncio/tasks.py", line 237, in _step
- result = next(coro)
- File "asyncio/coroutines.py", line 79, in __next__
- return next(self.gen)
- File "asyncio/coroutines.py", line 141, in coro
- res = func(*args, **kw)
- File "test.py", line 5, in bug
+ File "test.py", line 4, in bug
raise Exception("not consumed")
Exception: not consumed
-There are different options to fix this issue. The first option is to chain the
-coroutine in another coroutine and use classic try/except::
-
- @asyncio.coroutine
- def handle_exception():
- try:
- yield from bug()
- except Exception:
- print("exception consumed")
-
- loop = asyncio.get_event_loop()
- asyncio.ensure_future(handle_exception())
- loop.run_forever()
- loop.close()
-
-Another option is to use the :meth:`AbstractEventLoop.run_until_complete`
-function::
-
- task = asyncio.ensure_future(bug())
- try:
- loop.run_until_complete(task)
- except Exception:
- print("exception consumed")
-
-.. seealso::
-
- The :meth:`Future.exception` method.
-
-
-Chain coroutines correctly
---------------------------
-
-When a coroutine function calls other coroutine functions and tasks, they
-should be chained explicitly with ``yield from``. Otherwise, the execution is
-not guaranteed to be sequential.
+:ref:`Enable the debug mode ` to get the
+traceback where the task was created::
-Example with different bugs using :func:`asyncio.sleep` to simulate slow
-operations::
+ asyncio.run(main(), debug=True)
- import asyncio
-
- @asyncio.coroutine
- def create():
- yield from asyncio.sleep(3.0)
- print("(1) create file")
-
- @asyncio.coroutine
- def write():
- yield from asyncio.sleep(1.0)
- print("(2) write into file")
-
- @asyncio.coroutine
- def close():
- print("(3) close file")
-
- @asyncio.coroutine
- def test():
- asyncio.ensure_future(create())
- asyncio.ensure_future(write())
- asyncio.ensure_future(close())
- yield from asyncio.sleep(2.0)
- loop.stop()
-
- loop = asyncio.get_event_loop()
- asyncio.ensure_future(test())
- loop.run_forever()
- print("Pending tasks at exit: %s" % asyncio.Task.all_tasks(loop))
- loop.close()
-
-Expected output:
-
-.. code-block:: none
-
- (1) create file
- (2) write into file
- (3) close file
- Pending tasks at exit: set()
-
-Actual output:
-
-.. code-block:: none
-
- (3) close file
- (2) write into file
- Pending tasks at exit: {>}
- Task was destroyed but it is pending!
- task: >
-
-The loop stopped before the ``create()`` finished, ``close()`` has been called
-before ``write()``, whereas coroutine functions were called in this order:
-``create()``, ``write()``, ``close()``.
-
-To fix the example, tasks must be marked with ``yield from``::
-
- @asyncio.coroutine
- def test():
- yield from asyncio.ensure_future(create())
- yield from asyncio.ensure_future(write())
- yield from asyncio.ensure_future(close())
- yield from asyncio.sleep(2.0)
- loop.stop()
-
-Or without ``asyncio.ensure_future()``::
-
- @asyncio.coroutine
- def test():
- yield from create()
- yield from write()
- yield from close()
- yield from asyncio.sleep(2.0)
- loop.stop()
-
-
-.. _asyncio-pending-task-destroyed:
-
-Pending task destroyed
-----------------------
-
-If a pending task is destroyed, the execution of its wrapped :ref:`coroutine
-` did not complete. It is probably a bug and so a warning is logged.
-
-Example of log:
-
-.. code-block:: none
-
- Task was destroyed but it is pending!
- task: wait_for=>
-
-:ref:`Enable the debug mode of asyncio ` to get the
-traceback where the task was created. Example of log in debug mode:
+Output in debug mode::
-.. code-block:: none
+ Task exception was never retrieved
+ future:
+ exception=Exception('not consumed') created at asyncio/tasks.py:321>
- Task was destroyed but it is pending!
source_traceback: Object created at (most recent call last):
- File "test.py", line 15, in
- task = asyncio.ensure_future(coro, loop=loop)
- task: wait_for= created at test.py:15>
+ File "../t.py", line 9, in
+ asyncio.run(main(), debug=True)
+ < .. >
-.. seealso::
-
- :ref:`Detect coroutine objects never scheduled `.
-
-.. _asyncio-close-transports:
-
-Close transports and event loops
---------------------------------
-
-When a transport is no more needed, call its ``close()`` method to release
-resources. Event loops must also be closed explicitly.
-
-If a transport or an event loop is not closed explicitly, a
-:exc:`ResourceWarning` warning will be emitted in its destructor. By default,
-:exc:`ResourceWarning` warnings are ignored. The :ref:`Debug mode of asyncio
-` section explains how to display them.
+ Traceback (most recent call last):
+ File "../t.py", line 4, in bug
+ raise Exception("not consumed")
+ Exception: not consumed
diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
index bdba9962df62273ab467983e11282a9f89d428f2..647b7fc5e7a5820633fef9ea15fe4b233e13fd8f 100644
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -1,103 +1,163 @@
.. currentmodule:: asyncio
-.. _asyncio-event-loop:
-Base Event Loop
-===============
+==========
+Event Loop
+==========
-**Source code:** :source:`Lib/asyncio/events.py`
-The event loop is the central execution device provided by :mod:`asyncio`.
-It provides multiple facilities, including:
+.. rubric:: Preface
-* Registering, executing and cancelling delayed calls (timeouts).
+The event loop is the core of every asyncio application.
+Event loops run asynchronous tasks and callbacks, perform network
+IO operations, and run subprocesses.
-* Creating client and server :ref:`transports ` for various
- kinds of communication.
+Application developers should typically use the high-level asyncio functions,
+such as :func:`asyncio.run`, and should rarely need to reference the loop
+object or call its methods. This section is intended mostly for authors
+of lower-level code, libraries, and frameworks, who need finer control over
+the event loop behavior.
-* Launching subprocesses and the associated :ref:`transports
- ` for communication with an external program.
+.. rubric:: Obtaining the Event Loop
-* Delegating costly function calls to a pool of threads.
+The following low-level functions can be used to get, set, or create
+an event loop:
-.. class:: BaseEventLoop
+.. function:: get_running_loop()
- This class is an implementation detail. It is a subclass of
- :class:`AbstractEventLoop` and may be a base class of concrete
- event loop implementations found in :mod:`asyncio`. It should not
- be used directly; use :class:`AbstractEventLoop` instead.
- ``BaseEventLoop`` should not be subclassed by third-party code; the
- internal interface is not stable.
+ Return the running event loop in the current OS thread.
-.. class:: AbstractEventLoop
+ If there is no running event loop a :exc:`RuntimeError` is raised.
+ This function can only be called from a coroutine or a callback.
- Abstract base class of event loops.
+ .. versionadded:: 3.7
- This class is :ref:`not thread safe `.
+.. function:: get_event_loop()
-Run an event loop
------------------
+ Get the current event loop. If there is no current event loop set
+ in the current OS thread and :func:`set_event_loop` has not yet
+ been called, asyncio will create a new event loop and set it as the
+ current one.
-.. method:: AbstractEventLoop.run_forever()
+ Because this function has rather complex behavior (especially
+ when custom event loop policies are in use), using the
+ :func:`get_running_loop` function is preferred to :func:`get_event_loop`
+ in coroutines and callbacks.
- Run until :meth:`stop` is called. If :meth:`stop` is called before
- :meth:`run_forever()` is called, this polls the I/O selector once
- with a timeout of zero, runs all callbacks scheduled in response to
- I/O events (and those that were already scheduled), and then exits.
- If :meth:`stop` is called while :meth:`run_forever` is running,
- this will run the current batch of callbacks and then exit. Note
- that callbacks scheduled by callbacks will not run in that case;
- they will run the next time :meth:`run_forever` is called.
+ Consider also using the :func:`asyncio.run` function instead of using
+ lower level functions to manually create and close an event loop.
- .. versionchanged:: 3.5.1
+.. function:: set_event_loop(loop)
-.. method:: AbstractEventLoop.run_until_complete(future)
+ Set *loop* as a current event loop for the current OS thread.
- Run until the :class:`Future` is done.
+.. function:: new_event_loop()
- If the argument is a :ref:`coroutine object `, it is wrapped by
- :func:`ensure_future`.
+ Create a new event loop object.
- Return the Future's result, or raise its exception.
+Note that the behaviour of :func:`get_event_loop`, :func:`set_event_loop`,
+and :func:`new_event_loop` functions can be altered by
+:ref:`setting a custom event loop policy `.
-.. method:: AbstractEventLoop.is_running()
- Returns running status of event loop.
+.. rubric:: Contents
-.. method:: AbstractEventLoop.stop()
+This documentation page contains the following sections:
- Stop running the event loop.
+* The `Event Loop Methods`_ section is the reference documentation of
+ the event loop APIs;
- This causes :meth:`run_forever` to exit at the next suitable
- opportunity (see there for more details).
+* The `Callback Handles`_ section documents the :class:`Handle` and
+ :class:`TimerHandle` instances which are returned from scheduling
+ methods such as :meth:`loop.call_soon` and :meth:`loop.call_later`;
+
+* The `Server Objects`_ section documents types returned from
+ event loop methods like :meth:`loop.create_server`;
+
+* The `Event Loop Implementations`_ section documents the
+ :class:`SelectorEventLoop` and :class:`ProactorEventLoop` classes;
+
+* The `Examples`_ section showcases how to work with some event
+ loop APIs.
+
+
+.. _asyncio-event-loop:
+
+Event Loop Methods
+==================
+
+Event loops have **low-level** APIs for the following:
+
+.. contents::
+ :depth: 1
+ :local:
- .. versionchanged:: 3.5.1
-.. method:: AbstractEventLoop.is_closed()
+Running and stopping the loop
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- Returns ``True`` if the event loop was closed.
+.. method:: loop.run_until_complete(future)
- .. versionadded:: 3.4.2
+ Run until the *future* (an instance of :class:`Future`) has
+ completed.
-.. method:: AbstractEventLoop.close()
+ If the argument is a :ref:`coroutine object ` it
+ is implicitly scheduled to run as a :class:`asyncio.Task`.
- Close the event loop. The loop must not be running. Pending
- callbacks will be lost.
+ Return the Future's result or raise its exception.
- This clears the queues and shuts down the executor, but does not wait for
- the executor to finish.
+.. method:: loop.run_forever()
+
+ Run the event loop until :meth:`stop` is called.
+
+ If :meth:`stop` is called before :meth:`run_forever()` is called,
+ the loop will poll the I/O selector once with a timeout of zero,
+ run all callbacks scheduled in response to I/O events (and
+ those that were already scheduled), and then exit.
+
+ If :meth:`stop` is called while :meth:`run_forever` is running,
+ the loop will run the current batch of callbacks and then exit.
+ Note that new callbacks scheduled by callbacks will not run in this
+ case; instead, they will run the next time :meth:`run_forever` or
+ :meth:`run_until_complete` is called.
- This is idempotent and irreversible. No other methods should be called after
- this one.
+.. method:: loop.stop()
+ Stop the event loop.
-.. coroutinemethod:: AbstractEventLoop.shutdown_asyncgens()
+.. method:: loop.is_running()
+
+ Return ``True`` if the event loop is currently running.
+
+.. method:: loop.is_closed()
+
+ Return ``True`` if the event loop was closed.
+
+.. method:: loop.close()
+
+ Close the event loop.
+
+ The loop must not be running when this function is called.
+ Any pending callbacks will be discarded.
+
+ This method clears all queues and shuts down the executor, but does
+ not wait for the executor to finish.
+
+ This method is idempotent and irreversible. No other methods
+ should be called after the event loop is closed.
+
+.. coroutinemethod:: loop.shutdown_asyncgens()
Schedule all currently open :term:`asynchronous generator` objects to
close with an :meth:`~agen.aclose()` call. After calling this method,
- the event loop will issue a warning whenever a new asynchronous generator
- is iterated. Should be used to finalize all scheduled asynchronous
- generators reliably. Example::
+ the event loop will issue a warning if a new asynchronous generator
+ is iterated. This should be used to reliably finalize all scheduled
+ asynchronous generators.
+
+ Note that there is no need to call this function when
+ :func:`asyncio.run` is used.
+
+ Example::
try:
loop.run_forever()
@@ -108,212 +168,223 @@ Run an event loop
.. versionadded:: 3.6
-.. _asyncio-pass-keywords:
-
-Calls
------
-
-Most :mod:`asyncio` functions don't accept keywords. If you want to pass
-keywords to your callback, use :func:`functools.partial`. For example,
-``loop.call_soon(functools.partial(print, "Hello", flush=True))`` will call
-``print("Hello", flush=True)``.
-
-.. note::
- :func:`functools.partial` is better than ``lambda`` functions, because
- :mod:`asyncio` can inspect :func:`functools.partial` object to display
- parameters in debug mode, whereas ``lambda`` functions have a poor
- representation.
+Scheduling callbacks
+^^^^^^^^^^^^^^^^^^^^
-.. method:: AbstractEventLoop.call_soon(callback, \*args)
+.. method:: loop.call_soon(callback, *args, context=None)
- Arrange for a callback to be called as soon as possible. The callback is
- called after :meth:`call_soon` returns, when control returns to the event
- loop.
+ Schedule a *callback* to be called with *args* arguments at
+ the next iteration of the event loop.
- This operates as a :abbr:`FIFO (first-in, first-out)` queue, callbacks
- are called in the order in which they are registered. Each callback
- will be called exactly once.
+ Callbacks are called in the order in which they are registered.
+ Each callback will be called exactly once.
- Any positional arguments after the callback will be passed to the
- callback when it is called.
+ An optional keyword-only *context* argument allows specifying a
+ custom :class:`contextvars.Context` for the *callback* to run in.
+ The current context is used when no *context* is provided.
An instance of :class:`asyncio.Handle` is returned, which can be
- used to cancel the callback.
+ used later to cancel the callback.
- :ref:`Use functools.partial to pass keywords to the callback
- `.
+ This method is not thread-safe.
-.. method:: AbstractEventLoop.call_soon_threadsafe(callback, \*args)
+.. method:: loop.call_soon_threadsafe(callback, *args, context=None)
- Like :meth:`call_soon`, but thread safe.
+ A thread-safe variant of :meth:`call_soon`. Must be used to
+ schedule callbacks *from another thread*.
See the :ref:`concurrency and multithreading `
section of the documentation.
+.. versionchanged:: 3.7
+ The *context* keyword-only parameter was added. See :pep:`567`
+ for more details.
+
+.. _asyncio-pass-keywords:
+
+.. note::
+
+ Most :mod:`asyncio` scheduling functions don't allow passing
+ keyword arguments. To do that, use :func:`functools.partial`::
+
+ # will schedule "print("Hello", flush=True)"
+ loop.call_soon(
+ functools.partial(print, "Hello", flush=True))
+
+ Using partial objects is usually more convenient than using lambdas,
+ as asyncio can render partial objects better in debug and error
+ messages.
+
.. _asyncio-delayed-calls:
-Delayed calls
--------------
+Scheduling delayed callbacks
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-The event loop has its own internal clock for computing timeouts.
-Which clock is used depends on the (platform-specific) event loop
-implementation; ideally it is a monotonic clock. This will generally be
-a different clock than :func:`time.time`.
+Event loop provides mechanisms to schedule callback functions
+to be called at some point in the future. Event loop uses monotonic
+clocks to track time.
-.. note::
- Timeouts (relative *delay* or absolute *when*) should not exceed one day.
+.. method:: loop.call_later(delay, callback, *args, context=None)
+ Schedule *callback* to be called after the given *delay*
+ number of seconds (can be either an int or a float).
-.. method:: AbstractEventLoop.call_later(delay, callback, *args)
+ An instance of :class:`asyncio.TimerHandle` is returned which can
+ be used to cancel the callback.
- Arrange for the *callback* to be called after the given *delay*
- seconds (either an int or float).
+ *callback* will be called exactly once. If two callbacks are
+ scheduled for exactly the same time, the order in which they
+ are called is undefined.
- An instance of :class:`asyncio.Handle` is returned, which can be
- used to cancel the callback.
+ The optional positional *args* will be passed to the callback when
+ it is called. If you want the callback to be called with keyword
+ arguments use :func:`functools.partial`.
- *callback* will be called exactly once per call to :meth:`call_later`.
- If two callbacks are scheduled for exactly the same time, it is
- undefined which will be called first.
+ An optional keyword-only *context* argument allows specifying a
+ custom :class:`contextvars.Context` for the *callback* to run in.
+ The current context is used when no *context* is provided.
- The optional positional *args* will be passed to the callback when it
- is called. If you want the callback to be called with some named
- arguments, use a closure or :func:`functools.partial`.
+ .. versionchanged:: 3.7
+ The *context* keyword-only parameter was added. See :pep:`567`
+ for more details.
- :ref:`Use functools.partial to pass keywords to the callback
- `.
+ .. versionchanged:: 3.7.1
+ In Python 3.7.0 and earlier with the default event loop implementation,
+ the *delay* could not exceed one day.
+ This has been fixed in Python 3.7.1.
-.. method:: AbstractEventLoop.call_at(when, callback, *args)
+.. method:: loop.call_at(when, callback, *args, context=None)
- Arrange for the *callback* to be called at the given absolute timestamp
- *when* (an int or float), using the same time reference as
- :meth:`AbstractEventLoop.time`.
+ Schedule *callback* to be called at the given absolute timestamp
+ *when* (an int or a float), using the same time reference as
+ :meth:`loop.time`.
This method's behavior is the same as :meth:`call_later`.
- An instance of :class:`asyncio.Handle` is returned, which can be
- used to cancel the callback.
-
- :ref:`Use functools.partial to pass keywords to the callback
- `.
+ An instance of :class:`asyncio.TimerHandle` is returned which can
+ be used to cancel the callback.
-.. method:: AbstractEventLoop.time()
+ .. versionchanged:: 3.7
+ The *context* keyword-only parameter was added. See :pep:`567`
+ for more details.
- Return the current time, as a :class:`float` value, according to the
- event loop's internal clock.
+ .. versionchanged:: 3.7.1
+ In Python 3.7.0 and earlier with the default event loop implementation,
+ the difference between *when* and the current time could not exceed
+ one day. This has been fixed in Python 3.7.1.
-.. seealso::
+.. method:: loop.time()
- The :func:`asyncio.sleep` function.
+ Return the current time, as a :class:`float` value, according to
+ the event loop's internal monotonic clock.
+.. note::
-Futures
--------
+ Timeouts (relative *delay* or absolute *when*) should not
+ exceed one day.
-.. method:: AbstractEventLoop.create_future()
+.. seealso::
- Create an :class:`asyncio.Future` object attached to the loop.
+ The :func:`asyncio.sleep` function.
- This is a preferred way to create futures in asyncio, as event
- loop implementations can provide alternative implementations
- of the Future class (with better performance or instrumentation).
- .. versionadded:: 3.5.2
+Creating Futures and Tasks
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. method:: loop.create_future()
-Tasks
------
+ Create an :class:`asyncio.Future` object attached to the event loop.
-.. method:: AbstractEventLoop.create_task(coro)
+ This is the preferred way to create Futures in asyncio. This lets
+ third-party event loops provide alternative implementations of
+ the Future object (with better performance or instrumentation).
- Schedule the execution of a :ref:`coroutine object `: wrap it in
- a future. Return a :class:`Task` object.
+ .. versionadded:: 3.5.2
- Third-party event loops can use their own subclass of :class:`Task` for
- interoperability. In this case, the result type is a subclass of
- :class:`Task`.
+.. method:: loop.create_task(coro)
- This method was added in Python 3.4.2. Use the :func:`async` function to
- support also older Python versions.
+ Schedule the execution of a :ref:`coroutine`.
+ Return a :class:`Task` object.
- .. versionadded:: 3.4.2
+ Third-party event loops can use their own subclass of :class:`Task`
+ for interoperability. In this case, the result type is a subclass
+ of :class:`Task`.
-.. method:: AbstractEventLoop.set_task_factory(factory)
+.. method:: loop.set_task_factory(factory)
Set a task factory that will be used by
- :meth:`AbstractEventLoop.create_task`.
+ :meth:`loop.create_task`.
If *factory* is ``None`` the default task factory will be set.
+ Otherwise, *factory* must be a *callable* with the signature matching
+ ``(loop, coro)``, where *loop* is a reference to the active
+ event loop, and *coro* is a coroutine object. The callable
+ must return a :class:`asyncio.Future`-compatible object.
- If *factory* is a *callable*, it should have a signature matching
- ``(loop, coro)``, where *loop* will be a reference to the active
- event loop, *coro* will be a coroutine object. The callable
- must return an :class:`asyncio.Future` compatible object.
+.. method:: loop.get_task_factory()
- .. versionadded:: 3.4.4
+ Return a task factory or ``None`` if the default one is in use.
-.. method:: AbstractEventLoop.get_task_factory()
- Return a task factory, or ``None`` if the default one is in use.
+Opening network connections
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
- .. versionadded:: 3.4.4
+.. coroutinemethod:: loop.create_connection(protocol_factory, \
+ host=None, port=None, \*, ssl=None, \
+ family=0, proto=0, flags=0, sock=None, \
+ local_addr=None, server_hostname=None, \
+ ssl_handshake_timeout=None)
+ Open a streaming transport connection to a given
+ address specified by *host* and *port*.
-Creating connections
---------------------
+ The socket family can be either :py:data:`~socket.AF_INET` or
+ :py:data:`~socket.AF_INET6` depending on *host* (or the *family*
+ argument, if provided).
-.. coroutinemethod:: AbstractEventLoop.create_connection(protocol_factory, host=None, port=None, \*, ssl=None, family=0, proto=0, flags=0, sock=None, local_addr=None, server_hostname=None)
+ The socket type will be :py:data:`~socket.SOCK_STREAM`.
- Create a streaming transport connection to a given Internet *host* and
- *port*: socket family :py:data:`~socket.AF_INET` or
- :py:data:`~socket.AF_INET6` depending on *host* (or *family* if specified),
- socket type :py:data:`~socket.SOCK_STREAM`. *protocol_factory* must be a
- callable returning a :ref:`protocol ` instance.
+ *protocol_factory* must be a callable returning an
+ :ref:`asyncio protocol ` implementation.
- This method is a :ref:`coroutine ` which will try to
- establish the connection in the background. When successful, the
- coroutine returns a ``(transport, protocol)`` pair.
+ This method will try to establish the connection in the background.
+ When successful, it returns a ``(transport, protocol)`` pair.
The chronological synopsis of the underlying operation is as follows:
- #. The connection is established, and a :ref:`transport `
- is created to represent it.
+ #. The connection is established and a :ref:`transport `
+ is created for it.
- #. *protocol_factory* is called without arguments and must return a
- :ref:`protocol ` instance.
+ #. *protocol_factory* is called without arguments and is expected to
+ return a :ref:`protocol ` instance.
- #. The protocol instance is tied to the transport, and its
- :meth:`connection_made` method is called.
+ #. The protocol instance is coupled with the transport by calling its
+ :meth:`~BaseProtocol.connection_made` method.
- #. The coroutine returns successfully with the ``(transport, protocol)``
- pair.
+ #. A ``(transport, protocol)`` tuple is returned on success.
- The created transport is an implementation-dependent bidirectional stream.
+ The created transport is an implementation-dependent bidirectional
+ stream.
- .. note::
- *protocol_factory* can be any kind of callable, not necessarily
- a class. For example, if you want to use a pre-created
- protocol instance, you can pass ``lambda: my_protocol``.
-
- Options that change how the connection is created:
+ Other arguments:
* *ssl*: if given and not false, a SSL/TLS transport is created
(by default a plain TCP transport is created). If *ssl* is
a :class:`ssl.SSLContext` object, this context is used to create
- the transport; if *ssl* is :const:`True`, a context with some
- unspecified default settings is used.
+ the transport; if *ssl* is :const:`True`, a default context returned
+ from :func:`ssl.create_default_context` is used.
.. seealso:: :ref:`SSL/TLS security considerations `
- * *server_hostname*, is only for use together with *ssl*,
- and sets or overrides the hostname that the target server's certificate
- will be matched against. By default the value of the *host* argument
+ * *server_hostname* sets or overrides the hostname that the target
+ server's certificate will be matched against. Should only be passed
+ if *ssl* is not ``None``. By default the value of the *host* argument
is used. If *host* is empty, there is no default and you must pass a
value for *server_hostname*. If *server_hostname* is an empty
string, hostname matching is disabled (which is a serious security
- risk, allowing for man-in-the-middle-attacks).
+ risk, allowing for potential man-in-the-middle attacks).
* *family*, *proto*, *flags* are the optional address family, protocol
and flags to be passed through to getaddrinfo() for *host* resolution.
@@ -327,30 +398,51 @@ Creating connections
* *local_addr*, if given, is a ``(local_host, local_port)`` tuple used
to bind the socket to locally. The *local_host* and *local_port*
- are looked up using getaddrinfo(), similarly to *host* and *port*.
+ are looked up using ``getaddrinfo()``, similarly to *host* and *port*.
+
+ * *ssl_handshake_timeout* is (for a TLS connection) the time in seconds
+ to wait for the TLS handshake to complete before aborting the connection.
+ ``60.0`` seconds if ``None`` (default).
+
+ .. versionadded:: 3.7
+
+ The *ssl_handshake_timeout* parameter.
+
+ .. versionchanged:: 3.6
+
+ The socket option :py:data:`~socket.TCP_NODELAY` is set by default
+ for all TCP connections.
.. versionchanged:: 3.5
- On Windows with :class:`ProactorEventLoop`, SSL/TLS is now supported.
+ Added support for SSL/TLS in :class:`ProactorEventLoop`.
.. seealso::
- The :func:`open_connection` function can be used to get a pair of
- (:class:`StreamReader`, :class:`StreamWriter`) instead of a protocol.
+ The :func:`open_connection` function is a high-level alternative
+ API. It returns a pair of (:class:`StreamReader`, :class:`StreamWriter`)
+ that can be used directly in async/await code.
+.. coroutinemethod:: loop.create_datagram_endpoint(protocol_factory, \
+ local_addr=None, remote_addr=None, \*, \
+ family=0, proto=0, flags=0, \
+ reuse_address=None, reuse_port=None, \
+ allow_broadcast=None, sock=None)
-.. coroutinemethod:: AbstractEventLoop.create_datagram_endpoint(protocol_factory, local_addr=None, remote_addr=None, \*, family=0, proto=0, flags=0, reuse_address=None, reuse_port=None, allow_broadcast=None, sock=None)
+ Create a datagram connection.
- Create datagram connection: socket family :py:data:`~socket.AF_INET` or
- :py:data:`~socket.AF_INET6` depending on *host* (or *family* if specified),
- socket type :py:data:`~socket.SOCK_DGRAM`. *protocol_factory* must be a
- callable returning a :ref:`protocol ` instance.
+ The socket family can be either :py:data:`~socket.AF_INET`,
+ :py:data:`~socket.AF_INET6`, or :py:data:`~socket.AF_UNIX`,
+ depending on *host* (or the *family* argument, if provided).
- This method is a :ref:`coroutine ` which will try to
- establish the connection in the background. When successful, the
- coroutine returns a ``(transport, protocol)`` pair.
+ The socket type will be :py:data:`~socket.SOCK_DGRAM`.
- Options changing how the connection is created:
+ *protocol_factory* must be a callable returning a
+ :ref:`protocol ` implementation.
+
+ A tuple of ``(transport, protocol)`` is returned on success.
+
+ Other arguments:
* *local_addr*, if given, is a ``(local_host, local_port)`` tuple used
to bind the socket to locally. The *local_host* and *local_port*
@@ -366,14 +458,14 @@ Creating connections
corresponding :mod:`socket` module constants.
* *reuse_address* tells the kernel to reuse a local socket in
- TIME_WAIT state, without waiting for its natural timeout to
+ ``TIME_WAIT`` state, without waiting for its natural timeout to
expire. If not specified will automatically be set to ``True`` on
- UNIX.
+ Unix.
* *reuse_port* tells the kernel to allow this endpoint to be bound to the
same port as other existing endpoints are bound to, so long as they all
set this flag when being created. This option is not supported on Windows
- and some UNIX's. If the :py:data:`~socket.SO_REUSEPORT` constant is not
+ and some Unixes. If the :py:data:`~socket.SO_REUSEPORT` constant is not
defined then this capability is unsupported.
* *allow_broadcast* tells the kernel to allow this endpoint to send
@@ -384,7 +476,7 @@ Creating connections
transport. If specified, *local_addr* and *remote_addr* should be omitted
(must be :const:`None`).
- On Windows with :class:`ProactorEventLoop`, this method is not supported.
+ On Windows, with :class:`ProactorEventLoop`, this method is not supported.
See :ref:`UDP echo client protocol ` and
:ref:`UDP echo server protocol ` examples.
@@ -393,216 +485,359 @@ Creating connections
The *family*, *proto*, *flags*, *reuse_address*, *reuse_port,
*allow_broadcast*, and *sock* parameters were added.
-.. coroutinemethod:: AbstractEventLoop.create_unix_connection(protocol_factory, path, \*, ssl=None, sock=None, server_hostname=None)
+.. coroutinemethod:: loop.create_unix_connection(protocol_factory, \
+ path=None, \*, ssl=None, sock=None, \
+ server_hostname=None, ssl_handshake_timeout=None)
- Create UNIX connection: socket family :py:data:`~socket.AF_UNIX`, socket
- type :py:data:`~socket.SOCK_STREAM`. The :py:data:`~socket.AF_UNIX` socket
- family is used to communicate between processes on the same machine
- efficiently.
+ Create a Unix connection.
- This method is a :ref:`coroutine ` which will try to
- establish the connection in the background. When successful, the
- coroutine returns a ``(transport, protocol)`` pair.
+ The socket family will be :py:data:`~socket.AF_UNIX`; socket
+ type will be :py:data:`~socket.SOCK_STREAM`.
- *path* is the name of a UNIX domain socket, and is required unless a *sock*
- parameter is specified. Abstract UNIX sockets, :class:`str`, and
- :class:`bytes` paths are supported.
+ A tuple of ``(transport, protocol)`` is returned on success.
- See the :meth:`AbstractEventLoop.create_connection` method for parameters.
+ *path* is the name of a Unix domain socket and is required,
+ unless a *sock* parameter is specified. Abstract Unix sockets,
+ :class:`str`, :class:`bytes`, and :class:`~pathlib.Path` paths are
+ supported.
- Availability: UNIX.
+ See the documentation of the :meth:`loop.create_connection` method
+ for information about arguments to this method.
+ .. availability:: Unix.
-Creating listening connections
-------------------------------
+ .. versionadded:: 3.7
-.. coroutinemethod:: AbstractEventLoop.create_server(protocol_factory, host=None, port=None, \*, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None, reuse_port=None)
+ The *ssl_handshake_timeout* parameter.
- Create a TCP server (socket type :data:`~socket.SOCK_STREAM`) bound to
- *host* and *port*.
+ .. versionchanged:: 3.7
- Return a :class:`Server` object, its :attr:`~Server.sockets` attribute
- contains created sockets. Use the :meth:`Server.close` method to stop the
- server: close listening sockets.
+ The *path* parameter can now be a :term:`path-like object`.
- Parameters:
- * The *host* parameter can be a string, in that case the TCP server is
- bound to *host* and *port*. The *host* parameter can also be a sequence
- of strings and in that case the TCP server is bound to all hosts of the
- sequence. If *host* is an empty string or ``None``, all interfaces are
- assumed and a list of multiple sockets will be returned (most likely one
- for IPv4 and another one for IPv6).
+Creating network servers
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. coroutinemethod:: loop.create_server(protocol_factory, \
+ host=None, port=None, \*, \
+ family=socket.AF_UNSPEC, \
+ flags=socket.AI_PASSIVE, \
+ sock=None, backlog=100, ssl=None, \
+ reuse_address=None, reuse_port=None, \
+ ssl_handshake_timeout=None, start_serving=True)
+
+ Create a TCP server (socket type :data:`~socket.SOCK_STREAM`) listening
+ on *port* of the *host* address.
+
+ Returns a :class:`Server` object.
+
+ Arguments:
+
+ * *protocol_factory* must be a callable returning a
+ :ref:`protocol ` implementation.
+
+ * The *host* parameter can be set to several types which determine where
+ the server would be listening:
+
+ - If *host* is a string, the TCP server is bound to a single network
+ interface specified by *host*.
+
+ - If *host* is a sequence of strings, the TCP server is bound to all
+ network interfaces specified by the sequence.
+
+ - If *host* is an empty string or ``None``, all interfaces are
+ assumed and a list of multiple sockets will be returned (most likely
+ one for IPv4 and another one for IPv6).
* *family* can be set to either :data:`socket.AF_INET` or
- :data:`~socket.AF_INET6` to force the socket to use IPv4 or IPv6. If not set
- it will be determined from host (defaults to :data:`socket.AF_UNSPEC`).
+ :data:`~socket.AF_INET6` to force the socket to use IPv4 or IPv6.
+ If not set, the *family* will be determined from host name
+ (defaults to :data:`~socket.AF_UNSPEC`).
* *flags* is a bitmask for :meth:`getaddrinfo`.
* *sock* can optionally be specified in order to use a preexisting
- socket object. If specified, *host* and *port* should be omitted (must be
- :const:`None`).
+ socket object. If specified, *host* and *port* must not be specified.
* *backlog* is the maximum number of queued connections passed to
:meth:`~socket.socket.listen` (defaults to 100).
- * *ssl* can be set to an :class:`~ssl.SSLContext` to enable SSL over the
- accepted connections.
+ * *ssl* can be set to an :class:`~ssl.SSLContext` instance to enable
+ TLS over the accepted connections.
* *reuse_address* tells the kernel to reuse a local socket in
- TIME_WAIT state, without waiting for its natural timeout to
+ ``TIME_WAIT`` state, without waiting for its natural timeout to
expire. If not specified will automatically be set to ``True`` on
- UNIX.
+ Unix.
* *reuse_port* tells the kernel to allow this endpoint to be bound to the
same port as other existing endpoints are bound to, so long as they all
set this flag when being created. This option is not supported on
Windows.
- This method is a :ref:`coroutine `.
+ * *ssl_handshake_timeout* is (for a TLS server) the time in seconds to wait
+ for the TLS handshake to complete before aborting the connection.
+ ``60.0`` seconds if ``None`` (default).
+
+ * *start_serving* set to ``True`` (the default) causes the created server
+ to start accepting connections immediately. When set to ``False``,
+ the user should await on :meth:`Server.start_serving` or
+ :meth:`Server.serve_forever` to make the server to start accepting
+ connections.
+
+ .. versionadded:: 3.7
+
+ Added *ssl_handshake_timeout* and *start_serving* parameters.
+
+ .. versionchanged:: 3.6
+
+ The socket option :py:data:`~socket.TCP_NODELAY` is set by default
+ for all TCP connections.
.. versionchanged:: 3.5
- On Windows with :class:`ProactorEventLoop`, SSL/TLS is now supported.
+ Added support for SSL/TLS in :class:`ProactorEventLoop`.
+
+ .. versionchanged:: 3.5.1
+
+ The *host* parameter can be a sequence of strings.
.. seealso::
- The function :func:`start_server` creates a (:class:`StreamReader`,
- :class:`StreamWriter`) pair and calls back a function with this pair.
+ The :func:`start_server` function is a higher-level alternative API
+ that returns a pair of :class:`StreamReader` and :class:`StreamWriter`
+ that can be used in an async/await code.
- .. versionchanged:: 3.5.1
- The *host* parameter can now be a sequence of strings.
+.. coroutinemethod:: loop.create_unix_server(protocol_factory, path=None, \
+ \*, sock=None, backlog=100, ssl=None, \
+ ssl_handshake_timeout=None, start_serving=True)
+
+ Similar to :meth:`loop.create_server` but works with the
+ :py:data:`~socket.AF_UNIX` socket family.
+ *path* is the name of a Unix domain socket, and is required,
+ unless a *sock* argument is provided. Abstract Unix sockets,
+ :class:`str`, :class:`bytes`, and :class:`~pathlib.Path` paths
+ are supported.
-.. coroutinemethod:: AbstractEventLoop.create_unix_server(protocol_factory, path=None, \*, sock=None, backlog=100, ssl=None)
+ See the documentation of the :meth:`loop.create_server` method
+ for information about arguments to this method.
- Similar to :meth:`AbstractEventLoop.create_server`, but specific to the
- socket family :py:data:`~socket.AF_UNIX`.
+ .. availability:: Unix.
- This method is a :ref:`coroutine `.
+ .. versionadded:: 3.7
- Availability: UNIX.
+ The *ssl_handshake_timeout* and *start_serving* parameters.
-.. coroutinemethod:: BaseEventLoop.connect_accepted_socket(protocol_factory, sock, \*, ssl=None)
+ .. versionchanged:: 3.7
- Handle an accepted connection.
+ The *path* parameter can now be a :class:`~pathlib.Path` object.
- This is used by servers that accept connections outside of
- asyncio but that use asyncio to handle them.
+.. coroutinemethod:: loop.connect_accepted_socket(protocol_factory, \
+ sock, \*, ssl=None, ssl_handshake_timeout=None)
+
+ Wrap an already accepted connection into a transport/protocol pair.
+
+ This method can be used by servers that accept connections outside
+ of asyncio but that use asyncio to handle them.
Parameters:
- * *sock* is a preexisting socket object returned from an ``accept``
- call.
+ * *protocol_factory* must be a callable returning a
+ :ref:`protocol ` implementation.
+
+ * *sock* is a preexisting socket object returned from
+ :meth:`socket.accept `.
+
+ * *ssl* can be set to an :class:`~ssl.SSLContext` to enable SSL over
+ the accepted connections.
- * *ssl* can be set to an :class:`~ssl.SSLContext` to enable SSL over the
- accepted connections.
+ * *ssl_handshake_timeout* is (for an SSL connection) the time in seconds to
+ wait for the SSL handshake to complete before aborting the connection.
+ ``60.0`` seconds if ``None`` (default).
- This method is a :ref:`coroutine `. When completed, the
- coroutine returns a ``(transport, protocol)`` pair.
+ Returns a ``(transport, protocol)`` pair.
+
+ .. versionadded:: 3.7
+
+ The *ssl_handshake_timeout* parameter.
.. versionadded:: 3.5.3
-Watch file descriptors
-----------------------
+Transferring files
+^^^^^^^^^^^^^^^^^^
+
+.. coroutinemethod:: loop.sendfile(transport, file, \
+ offset=0, count=None, *, fallback=True)
+
+ Send a *file* over a *transport*. Return the total number of bytes
+ sent.
+
+ The method uses high-performance :meth:`os.sendfile` if available.
+
+ *file* must be a regular file object opened in binary mode.
+
+ *offset* tells from where to start reading the file. If specified,
+ *count* is the total number of bytes to transmit as opposed to
+ sending the file until EOF is reached. File position is always updated,
+ even when this method raises an error, and
+ :meth:`file.tell() ` can be used to obtain the actual
+ number of bytes sent.
+
+ *fallback* set to ``True`` makes asyncio to manually read and send
+ the file when the platform does not support the sendfile system call
+ (e.g. Windows or SSL socket on Unix).
+
+ Raise :exc:`SendfileNotAvailableError` if the system does not support
+ the *sendfile* syscall and *fallback* is ``False``.
+
+ .. versionadded:: 3.7
+
+
+TLS Upgrade
+^^^^^^^^^^^
+
+.. coroutinemethod:: loop.start_tls(transport, protocol, \
+ sslcontext, \*, server_side=False, \
+ server_hostname=None, ssl_handshake_timeout=None)
+
+ Upgrade an existing transport-based connection to TLS.
+
+ Return a new transport instance, that the *protocol* must start using
+ immediately after the *await*. The *transport* instance passed to
+ the *start_tls* method should never be used again.
+
+ Parameters:
-On Windows with :class:`SelectorEventLoop`, only socket handles are supported
-(ex: pipe file descriptors are not supported).
+ * *transport* and *protocol* instances that methods like
+ :meth:`~loop.create_server` and
+ :meth:`~loop.create_connection` return.
-On Windows with :class:`ProactorEventLoop`, these methods are not supported.
+ * *sslcontext*: a configured instance of :class:`~ssl.SSLContext`.
-.. method:: AbstractEventLoop.add_reader(fd, callback, \*args)
+ * *server_side* pass ``True`` when a server-side connection is being
+ upgraded (like the one created by :meth:`~loop.create_server`).
- Start watching the file descriptor for read availability and then call the
- *callback* with specified arguments.
+ * *server_hostname*: sets or overrides the host name that the target
+ server's certificate will be matched against.
- :ref:`Use functools.partial to pass keywords to the callback
- `.
+ * *ssl_handshake_timeout* is (for a TLS connection) the time in seconds to
+ wait for the TLS handshake to complete before aborting the connection.
+ ``60.0`` seconds if ``None`` (default).
-.. method:: AbstractEventLoop.remove_reader(fd)
+ .. versionadded:: 3.7
- Stop watching the file descriptor for read availability.
-.. method:: AbstractEventLoop.add_writer(fd, callback, \*args)
+Watching file descriptors
+^^^^^^^^^^^^^^^^^^^^^^^^^
- Start watching the file descriptor for write availability and then call the
- *callback* with specified arguments.
+.. method:: loop.add_reader(fd, callback, \*args)
- :ref:`Use functools.partial to pass keywords to the callback
- `.
+ Start monitoring the *fd* file descriptor for read availability and
+ invoke *callback* with the specified arguments once *fd* is available for
+ reading.
-.. method:: AbstractEventLoop.remove_writer(fd)
+.. method:: loop.remove_reader(fd)
- Stop watching the file descriptor for write availability.
+ Stop monitoring the *fd* file descriptor for read availability.
-The :ref:`watch a file descriptor for read events `
-example uses the low-level :meth:`AbstractEventLoop.add_reader` method to register
-the file descriptor of a socket.
+.. method:: loop.add_writer(fd, callback, \*args)
+ Start monitoring the *fd* file descriptor for write availability and
+ invoke *callback* with the specified arguments once *fd* is available for
+ writing.
-Low-level socket operations
----------------------------
+ Use :func:`functools.partial` :ref:`to pass keyword arguments
+ ` to *callback*.
-.. coroutinemethod:: AbstractEventLoop.sock_recv(sock, nbytes)
+.. method:: loop.remove_writer(fd)
- Receive data from the socket. Modeled after blocking
- :meth:`socket.socket.recv` method.
+ Stop monitoring the *fd* file descriptor for write availability.
- The return value is a bytes object
- representing the data received. The maximum amount of data to be received
- at once is specified by *nbytes*.
+See also :ref:`Platform Support ` section
+for some limitations of these methods.
- With :class:`SelectorEventLoop` event loop, the socket *sock* must be
- non-blocking.
- This method is a :ref:`coroutine `.
+Working with socket objects directly
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-.. coroutinemethod:: AbstractEventLoop.sock_sendall(sock, data)
+In general, protocol implementations that use transport-based APIs
+such as :meth:`loop.create_connection` and :meth:`loop.create_server`
+are faster than implementations that work with sockets directly.
+However, there are some use cases when performance is not critical, and
+working with :class:`~socket.socket` objects directly is more
+convenient.
- Send data to the socket. Modeled after blocking
- :meth:`socket.socket.sendall` method.
+.. coroutinemethod:: loop.sock_recv(sock, nbytes)
- The socket must be connected to a remote socket.
- This method continues to send data from *data* until either all data has
- been sent or an error occurs. ``None`` is returned on success. On error,
- an exception is raised, and there is no way to determine how much data, if
- any, was successfully processed by the receiving end of the connection.
+ Receive up to *nbytes* from *sock*. Asynchronous version of
+ :meth:`socket.recv() `.
- With :class:`SelectorEventLoop` event loop, the socket *sock* must be
- non-blocking.
+ Return the received data as a bytes object.
- This method is a :ref:`coroutine `.
+ *sock* must be a non-blocking socket.
-.. coroutinemethod:: AbstractEventLoop.sock_connect(sock, address)
+ .. versionchanged:: 3.7
+ Even though this method was always documented as a coroutine
+ method, releases before Python 3.7 returned a :class:`Future`.
+ Since Python 3.7 this is an ``async def`` method.
- Connect to a remote socket at *address*. Modeled after
- blocking :meth:`socket.socket.connect` method.
+.. coroutinemethod:: loop.sock_recv_into(sock, buf)
- With :class:`SelectorEventLoop` event loop, the socket *sock* must be
- non-blocking.
+ Receive data from *sock* into the *buf* buffer. Modeled after the blocking
+ :meth:`socket.recv_into() ` method.
- This method is a :ref:`coroutine `.
+ Return the number of bytes written to the buffer.
+
+ *sock* must be a non-blocking socket.
+
+ .. versionadded:: 3.7
+
+.. coroutinemethod:: loop.sock_sendall(sock, data)
+
+ Send *data* to the *sock* socket. Asynchronous version of
+ :meth:`socket.sendall() `.
+
+ This method continues to send to the socket until either all data
+ in *data* has been sent or an error occurs. ``None`` is returned
+ on success. On error, an exception is raised. Additionally, there is no way
+ to determine how much data, if any, was successfully processed by the
+ receiving end of the connection.
+
+ *sock* must be a non-blocking socket.
+
+ .. versionchanged:: 3.7
+ Even though the method was always documented as a coroutine
+ method, before Python 3.7 it returned an :class:`Future`.
+ Since Python 3.7, this is an ``async def`` method.
+
+.. coroutinemethod:: loop.sock_connect(sock, address)
+
+ Connect *sock* to a remote socket at *address*.
+
+ Asynchronous version of :meth:`socket.connect() `.
+
+ *sock* must be a non-blocking socket.
.. versionchanged:: 3.5.2
``address`` no longer needs to be resolved. ``sock_connect``
will try to check if the *address* is already resolved by calling
:func:`socket.inet_pton`. If not,
- :meth:`AbstractEventLoop.getaddrinfo` will be used to resolve the
+ :meth:`loop.getaddrinfo` will be used to resolve the
*address*.
.. seealso::
- :meth:`AbstractEventLoop.create_connection`
+ :meth:`loop.create_connection`
and :func:`asyncio.open_connection() `.
-.. coroutinemethod:: AbstractEventLoop.sock_accept(sock)
+.. coroutinemethod:: loop.sock_accept(sock)
- Accept a connection. Modeled after blocking
- :meth:`socket.socket.accept`.
+ Accept a connection. Modeled after the blocking
+ :meth:`socket.accept() ` method.
The socket must be bound to an address and listening
for connections. The return value is a pair ``(conn, address)`` where *conn*
@@ -610,168 +845,260 @@ Low-level socket operations
and *address* is the address bound to the socket on the other end of the
connection.
- The socket *sock* must be non-blocking.
+ *sock* must be a non-blocking socket.
- This method is a :ref:`coroutine `.
+ .. versionchanged:: 3.7
+ Even though the method was always documented as a coroutine
+ method, before Python 3.7 it returned a :class:`Future`.
+ Since Python 3.7, this is an ``async def`` method.
.. seealso::
- :meth:`AbstractEventLoop.create_server` and :func:`start_server`.
+ :meth:`loop.create_server` and :func:`start_server`.
+
+.. coroutinemethod:: loop.sock_sendfile(sock, file, offset=0, count=None, \
+ \*, fallback=True)
+
+ Send a file using high-performance :mod:`os.sendfile` if possible.
+ Return the total number of bytes sent.
+
+ Asynchronous version of :meth:`socket.sendfile() `.
+
+ *sock* must be a non-blocking :const:`socket.SOCK_STREAM`
+ :class:`~socket.socket`.
+
+ *file* must be a regular file object open in binary mode.
+
+ *offset* tells from where to start reading the file. If specified,
+ *count* is the total number of bytes to transmit as opposed to
+ sending the file until EOF is reached. File position is always updated,
+ even when this method raises an error, and
+ :meth:`file.tell() ` can be used to obtain the actual
+ number of bytes sent.
+ *fallback*, when set to ``True``, makes asyncio manually read and send
+ the file when the platform does not support the sendfile syscall
+ (e.g. Windows or SSL socket on Unix).
-Resolve host name
------------------
+ Raise :exc:`SendfileNotAvailableError` if the system does not support
+ *sendfile* syscall and *fallback* is ``False``.
-.. coroutinemethod:: AbstractEventLoop.getaddrinfo(host, port, \*, family=0, type=0, proto=0, flags=0)
+ *sock* must be a non-blocking socket.
- This method is a :ref:`coroutine `, similar to
- :meth:`socket.getaddrinfo` function but non-blocking.
+ .. versionadded:: 3.7
-.. coroutinemethod:: AbstractEventLoop.getnameinfo(sockaddr, flags=0)
- This method is a :ref:`coroutine `, similar to
- :meth:`socket.getnameinfo` function but non-blocking.
+DNS
+^^^
+.. coroutinemethod:: loop.getaddrinfo(host, port, \*, family=0, \
+ type=0, proto=0, flags=0)
-Connect pipes
--------------
+ Asynchronous version of :meth:`socket.getaddrinfo`.
-On Windows with :class:`SelectorEventLoop`, these methods are not supported.
-Use :class:`ProactorEventLoop` to support pipes on Windows.
+.. coroutinemethod:: loop.getnameinfo(sockaddr, flags=0)
-.. coroutinemethod:: AbstractEventLoop.connect_read_pipe(protocol_factory, pipe)
+ Asynchronous version of :meth:`socket.getnameinfo`.
- Register read pipe in eventloop.
+.. versionchanged:: 3.7
+ Both *getaddrinfo* and *getnameinfo* methods were always documented
+ to return a coroutine, but prior to Python 3.7 they were, in fact,
+ returning :class:`asyncio.Future` objects. Starting with Python 3.7
+ both methods are coroutines.
- *protocol_factory* should instantiate object with :class:`Protocol`
- interface. *pipe* is a :term:`file-like object `.
- Return pair ``(transport, protocol)``, where *transport* supports the
- :class:`ReadTransport` interface.
+
+Working with pipes
+^^^^^^^^^^^^^^^^^^
+
+.. coroutinemethod:: loop.connect_read_pipe(protocol_factory, pipe)
+
+ Register the read end of *pipe* in the event loop.
+
+ *protocol_factory* must be a callable returning an
+ :ref:`asyncio protocol ` implementation.
+
+ *pipe* is a :term:`file-like object `.
+
+ Return pair ``(transport, protocol)``, where *transport* supports
+ the :class:`ReadTransport` interface and *protocol* is an object
+ instantiated by the *protocol_factory*.
With :class:`SelectorEventLoop` event loop, the *pipe* is set to
non-blocking mode.
- This method is a :ref:`coroutine `.
+.. coroutinemethod:: loop.connect_write_pipe(protocol_factory, pipe)
-.. coroutinemethod:: AbstractEventLoop.connect_write_pipe(protocol_factory, pipe)
+ Register the write end of *pipe* in the event loop.
- Register write pipe in eventloop.
+ *protocol_factory* must be a callable returning an
+ :ref:`asyncio protocol ` implementation.
+
+ *pipe* is :term:`file-like object `.
- *protocol_factory* should instantiate object with :class:`BaseProtocol`
- interface. *pipe* is :term:`file-like object `.
Return pair ``(transport, protocol)``, where *transport* supports
- :class:`WriteTransport` interface.
+ :class:`WriteTransport` interface and *protocol* is an object
+ instantiated by the *protocol_factory*.
With :class:`SelectorEventLoop` event loop, the *pipe* is set to
non-blocking mode.
- This method is a :ref:`coroutine `.
+.. note::
-.. seealso::
+ :class:`SelectorEventLoop` does not support the above methods on
+ Windows. Use :class:`ProactorEventLoop` instead for Windows.
- The :meth:`AbstractEventLoop.subprocess_exec` and
- :meth:`AbstractEventLoop.subprocess_shell` methods.
+.. seealso::
+ The :meth:`loop.subprocess_exec` and
+ :meth:`loop.subprocess_shell` methods.
-UNIX signals
-------------
-Availability: UNIX only.
+Unix signals
+^^^^^^^^^^^^
-.. method:: AbstractEventLoop.add_signal_handler(signum, callback, \*args)
+.. method:: loop.add_signal_handler(signum, callback, \*args)
- Add a handler for a signal.
+ Set *callback* as the handler for the *signum* signal.
Raise :exc:`ValueError` if the signal number is invalid or uncatchable.
Raise :exc:`RuntimeError` if there is a problem setting up the handler.
- :ref:`Use functools.partial to pass keywords to the callback
- `.
+ Use :func:`functools.partial` :ref:`to pass keyword arguments
+ ` to *callback*.
-.. method:: AbstractEventLoop.remove_signal_handler(sig)
+.. method:: loop.remove_signal_handler(sig)
- Remove a handler for a signal.
+ Remove the handler for the *sig* signal.
- Return ``True`` if a signal handler was removed, ``False`` if not.
+ Return ``True`` if the signal handler was removed, or ``False`` if
+ no handler was set for the given signal.
+
+ .. availability:: Unix.
.. seealso::
The :mod:`signal` module.
-Executor
---------
-
-Call a function in an :class:`~concurrent.futures.Executor` (pool of threads or
-pool of processes). By default, an event loop uses a thread pool executor
-(:class:`~concurrent.futures.ThreadPoolExecutor`).
+Executing code in thread or process pools
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-.. coroutinemethod:: AbstractEventLoop.run_in_executor(executor, func, \*args)
+.. awaitablemethod:: loop.run_in_executor(executor, func, \*args)
- Arrange for a *func* to be called in the specified executor.
+ Arrange for *func* to be called in the specified executor.
- The *executor* argument should be an :class:`~concurrent.futures.Executor`
+ The *executor* argument should be an :class:`concurrent.futures.Executor`
instance. The default executor is used if *executor* is ``None``.
- :ref:`Use functools.partial to pass keywords to the *func*
- `.
+ Example::
+
+ import asyncio
+ import concurrent.futures
+
+ def blocking_io():
+ # File operations (such as logging) can block the
+ # event loop: run them in a thread pool.
+ with open('/dev/urandom', 'rb') as f:
+ return f.read(100)
+
+ def cpu_bound():
+ # CPU-bound operations will block the event loop:
+ # in general it is preferable to run them in a
+ # process pool.
+ return sum(i * i for i in range(10 ** 7))
+
+ async def main():
+ loop = asyncio.get_running_loop()
- This method is a :ref:`coroutine `.
+ ## Options:
+
+ # 1. Run in the default loop's executor:
+ result = await loop.run_in_executor(
+ None, blocking_io)
+ print('default thread pool', result)
+
+ # 2. Run in a custom thread pool:
+ with concurrent.futures.ThreadPoolExecutor() as pool:
+ result = await loop.run_in_executor(
+ pool, blocking_io)
+ print('custom thread pool', result)
+
+ # 3. Run in a custom process pool:
+ with concurrent.futures.ProcessPoolExecutor() as pool:
+ result = await loop.run_in_executor(
+ pool, cpu_bound)
+ print('custom process pool', result)
+
+ asyncio.run(main())
+
+ This method returns a :class:`asyncio.Future` object.
+
+ Use :func:`functools.partial` :ref:`to pass keyword arguments
+ ` to *func*.
.. versionchanged:: 3.5.3
- :meth:`BaseEventLoop.run_in_executor` no longer configures the
+ :meth:`loop.run_in_executor` no longer configures the
``max_workers`` of the thread pool executor it creates, instead
leaving it up to the thread pool executor
(:class:`~concurrent.futures.ThreadPoolExecutor`) to set the
default.
-.. method:: AbstractEventLoop.set_default_executor(executor)
+.. method:: loop.set_default_executor(executor)
+
+ Set *executor* as the default executor used by :meth:`run_in_executor`.
+ *executor* should be an instance of
+ :class:`~concurrent.futures.ThreadPoolExecutor`.
- Set the default executor used by :meth:`run_in_executor`.
+ .. deprecated:: 3.7
+ Using an executor that is not an instance of
+ :class:`~concurrent.futures.ThreadPoolExecutor` is deprecated and
+ will trigger an error in Python 3.9.
+
+ *executor* must be an instance of
+ :class:`concurrent.futures.ThreadPoolExecutor`.
Error Handling API
-------------------
+^^^^^^^^^^^^^^^^^^
Allows customizing how exceptions are handled in the event loop.
-.. method:: AbstractEventLoop.set_exception_handler(handler)
+.. method:: loop.set_exception_handler(handler)
Set *handler* as the new event loop exception handler.
If *handler* is ``None``, the default exception handler will
- be set.
-
- If *handler* is a callable object, it should have a
- matching signature to ``(loop, context)``, where ``loop``
- will be a reference to the active event loop, ``context``
- will be a ``dict`` object (see :meth:`call_exception_handler`
- documentation for details about context).
+ be set. Otherwise, *handler* must be a callable with the signature
+ matching ``(loop, context)``, where ``loop``
+ is a reference to the active event loop, and ``context``
+ is a ``dict`` object containing the details of the exception
+ (see :meth:`call_exception_handler` documentation for details
+ about context).
-.. method:: AbstractEventLoop.get_exception_handler()
+.. method:: loop.get_exception_handler()
- Return the exception handler, or ``None`` if the default one
- is in use.
+ Return the current exception handler, or ``None`` if no custom
+ exception handler was set.
.. versionadded:: 3.5.2
-.. method:: AbstractEventLoop.default_exception_handler(context)
+.. method:: loop.default_exception_handler(context)
Default exception handler.
This is called when an exception occurs and no exception
- handler is set, and can be called by a custom exception
- handler that wants to defer to the default behavior.
+ handler is set. This can be called by a custom exception
+ handler that wants to defer to the default handler behavior.
*context* parameter has the same meaning as in
:meth:`call_exception_handler`.
-.. method:: AbstractEventLoop.call_exception_handler(context)
+.. method:: loop.call_exception_handler(context)
Call the current event loop exception handler.
*context* is a ``dict`` object containing the following keys
- (new keys may be introduced later):
+ (new keys may be introduced in future Python versions):
* 'message': Error message;
* 'exception' (optional): Exception object;
@@ -783,14 +1110,14 @@ Allows customizing how exceptions are handled in the event loop.
.. note::
- Note: this method should not be overloaded in subclassed
- event loops. For any custom exception handling, use
- :meth:`set_exception_handler()` method.
+ This method should not be overloaded in subclassed
+ event loops. For custom exception handling, use
+ the :meth:`set_exception_handler()` method.
-Debug mode
-----------
+Enabling debug mode
+^^^^^^^^^^^^^^^^^^^
-.. method:: AbstractEventLoop.get_debug()
+.. method:: loop.get_debug()
Get the debug mode (:class:`bool`) of the event loop.
@@ -798,81 +1125,354 @@ Debug mode
:envvar:`PYTHONASYNCIODEBUG` is set to a non-empty string, ``False``
otherwise.
- .. versionadded:: 3.4.2
-
-.. method:: AbstractEventLoop.set_debug(enabled: bool)
+.. method:: loop.set_debug(enabled: bool)
Set the debug mode of the event loop.
- .. versionadded:: 3.4.2
+ .. versionchanged:: 3.7
+
+ The new ``-X dev`` command line option can now also be used
+ to enable the debug mode.
.. seealso::
The :ref:`debug mode of asyncio `.
-Server
-------
+
+Running Subprocesses
+^^^^^^^^^^^^^^^^^^^^
+
+Methods described in this subsections are low-level. In regular
+async/await code consider using the high-level
+:func:`asyncio.create_subprocess_shell` and
+:func:`asyncio.create_subprocess_exec` convenience functions instead.
+
+.. note::
+
+ The default asyncio event loop on **Windows** does not support
+ subprocesses. See :ref:`Subprocess Support on Windows
+ ` for details.
+
+.. coroutinemethod:: loop.subprocess_exec(protocol_factory, \*args, \
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE, \*\*kwargs)
+
+ Create a subprocess from one or more string arguments specified by
+ *args*.
+
+ *args* must be a list of strings represented by:
+
+ * :class:`str`;
+ * or :class:`bytes`, encoded to the
+ :ref:`filesystem encoding `.
+
+ The first string specifies the program executable,
+ and the remaining strings specify the arguments. Together, string
+ arguments form the ``argv`` of the program.
+
+ This is similar to the standard library :class:`subprocess.Popen`
+ class called with ``shell=False`` and the list of strings passed as
+ the first argument; however, where :class:`~subprocess.Popen` takes
+ a single argument which is list of strings, *subprocess_exec*
+ takes multiple string arguments.
+
+ The *protocol_factory* must be a callable returning a subclass of the
+ :class:`asyncio.SubprocessProtocol` class.
+
+ Other parameters:
+
+ * *stdin*: either a file-like object representing a pipe to be
+ connected to the subprocess's standard input stream using
+ :meth:`~loop.connect_write_pipe`, or the
+ :const:`subprocess.PIPE` constant (default). By default a new
+ pipe will be created and connected.
+
+ * *stdout*: either a file-like object representing the pipe to be
+ connected to the subprocess's standard output stream using
+ :meth:`~loop.connect_read_pipe`, or the
+ :const:`subprocess.PIPE` constant (default). By default a new pipe
+ will be created and connected.
+
+ * *stderr*: either a file-like object representing the pipe to be
+ connected to the subprocess's standard error stream using
+ :meth:`~loop.connect_read_pipe`, or one of
+ :const:`subprocess.PIPE` (default) or :const:`subprocess.STDOUT`
+ constants.
+
+ By default a new pipe will be created and connected. When
+ :const:`subprocess.STDOUT` is specified, the subprocess' standard
+ error stream will be connected to the same pipe as the standard
+ output stream.
+
+ * All other keyword arguments are passed to :class:`subprocess.Popen`
+ without interpretation, except for *bufsize*, *universal_newlines*
+ and *shell*, which should not be specified at all.
+
+ See the constructor of the :class:`subprocess.Popen` class
+ for documentation on other arguments.
+
+ Returns a pair of ``(transport, protocol)``, where *transport*
+ conforms to the :class:`asyncio.SubprocessTransport` base class and
+ *protocol* is an object instantiated by the *protocol_factory*.
+
+.. coroutinemethod:: loop.subprocess_shell(protocol_factory, cmd, \*, \
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE, \*\*kwargs)
+
+ Create a subprocess from *cmd*, which can be a :class:`str` or a
+ :class:`bytes` string encoded to the
+ :ref:`filesystem encoding `,
+ using the platform's "shell" syntax.
+
+ This is similar to the standard library :class:`subprocess.Popen`
+ class called with ``shell=True``.
+
+ The *protocol_factory* must be a callable returning a subclass of the
+ :class:`SubprocessProtocol` class.
+
+ See :meth:`~loop.subprocess_exec` for more details about
+ the remaining arguments.
+
+ Returns a pair of ``(transport, protocol)``, where *transport*
+ conforms to the :class:`SubprocessTransport` base class and
+ *protocol* is an object instantiated by the *protocol_factory*.
+
+.. note::
+ It is the application's responsibility to ensure that all whitespace
+ and special characters are quoted appropriately to avoid `shell injection
+ `_
+ vulnerabilities. The :func:`shlex.quote` function can be used to
+ properly escape whitespace and special characters in strings that
+ are going to be used to construct shell commands.
+
+
+Callback Handles
+================
+
+.. class:: Handle
+
+ A callback wrapper object returned by :meth:`loop.call_soon`,
+ :meth:`loop.call_soon_threadsafe`.
+
+ .. method:: cancel()
+
+ Cancel the callback. If the callback has already been canceled
+ or executed, this method has no effect.
+
+ .. method:: cancelled()
+
+ Return ``True`` if the callback was cancelled.
+
+ .. versionadded:: 3.7
+
+.. class:: TimerHandle
+
+ A callback wrapper object returned by :meth:`loop.call_later`,
+ and :meth:`loop.call_at`.
+
+ This class is a subclass of :class:`Handle`.
+
+ .. method:: when()
+
+ Return a scheduled callback time as :class:`float` seconds.
+
+ The time is an absolute timestamp, using the same time
+ reference as :meth:`loop.time`.
+
+ .. versionadded:: 3.7
+
+
+Server Objects
+==============
+
+Server objects are created by :meth:`loop.create_server`,
+:meth:`loop.create_unix_server`, :func:`start_server`,
+and :func:`start_unix_server` functions.
+
+Do not instantiate the class directly.
.. class:: Server
- Server listening on sockets.
+ *Server* objects are asynchronous context managers. When used in an
+ ``async with`` statement, it's guaranteed that the Server object is
+ closed and not accepting new connections when the ``async with``
+ statement is completed::
- Object created by the :meth:`AbstractEventLoop.create_server` method and the
- :func:`start_server` function. Don't instantiate the class directly.
+ srv = await loop.create_server(...)
+
+ async with srv:
+ # some code
+
+ # At this point, srv is closed and no longer accepts new connections.
+
+
+ .. versionchanged:: 3.7
+ Server object is an asynchronous context manager since Python 3.7.
.. method:: close()
Stop serving: close listening sockets and set the :attr:`sockets`
attribute to ``None``.
- The sockets that represent existing incoming client connections are left
- open.
+ The sockets that represent existing incoming client connections
+ are left open.
The server is closed asynchronously, use the :meth:`wait_closed`
coroutine to wait until the server is closed.
+ .. method:: get_loop()
+
+ Return the event loop associated with the server object.
+
+ .. versionadded:: 3.7
+
+ .. coroutinemethod:: start_serving()
+
+ Start accepting connections.
+
+ This method is idempotent, so it can be called when
+ the server is already being serving.
+
+ The *start_serving* keyword-only parameter to
+ :meth:`loop.create_server` and
+ :meth:`asyncio.start_server` allows creating a Server object
+ that is not accepting connections initially. In this case
+ ``Server.start_serving()``, or :meth:`Server.serve_forever` can be used
+ to make the Server start accepting connections.
+
+ .. versionadded:: 3.7
+
+ .. coroutinemethod:: serve_forever()
+
+ Start accepting connections until the coroutine is cancelled.
+ Cancellation of ``serve_forever`` task causes the server
+ to be closed.
+
+ This method can be called if the server is already accepting
+ connections. Only one ``serve_forever`` task can exist per
+ one *Server* object.
+
+ Example::
+
+ async def client_connected(reader, writer):
+ # Communicate with the client with
+ # reader/writer streams. For example:
+ await reader.readline()
+
+ async def main(host, port):
+ srv = await asyncio.start_server(
+ client_connected, host, port)
+ await srv.serve_forever()
+
+ asyncio.run(main('127.0.0.1', 0))
+
+ .. versionadded:: 3.7
+
+ .. method:: is_serving()
+
+ Return ``True`` if the server is accepting new connections.
+
+ .. versionadded:: 3.7
+
.. coroutinemethod:: wait_closed()
Wait until the :meth:`close` method completes.
- This method is a :ref:`coroutine `.
-
.. attribute:: sockets
- List of :class:`socket.socket` objects the server is listening to, or
- ``None`` if the server is closed.
+ List of :class:`socket.socket` objects the server is listening on,
+ or ``None`` if the server is closed.
+ .. versionchanged:: 3.7
+ Prior to Python 3.7 ``Server.sockets`` used to return an
+ internal list of server sockets directly. In 3.7 a copy
+ of that list is returned.
-Handle
-------
-.. class:: Handle
+.. _asyncio-event-loops:
- A callback wrapper object returned by :func:`AbstractEventLoop.call_soon`,
- :func:`AbstractEventLoop.call_soon_threadsafe`, :func:`AbstractEventLoop.call_later`,
- and :func:`AbstractEventLoop.call_at`.
+Event Loop Implementations
+==========================
- .. method:: cancel()
+asyncio ships with two different event loop implementations:
+:class:`SelectorEventLoop` and :class:`ProactorEventLoop`.
- Cancel the call. If the callback is already canceled or executed,
- this method has no effect.
+By default asyncio is configured to use :class:`SelectorEventLoop`
+on all platforms.
-Event loop examples
--------------------
+.. class:: SelectorEventLoop
-.. _asyncio-hello-world-callback:
+ An event loop based on the :mod:`selectors` module.
+
+ Uses the most efficient *selector* available for the given
+ platform. It is also possible to manually configure the
+ exact selector implementation to be used::
+
+ import asyncio
+ import selectors
+
+ selector = selectors.SelectSelector()
+ loop = asyncio.SelectorEventLoop(selector)
+ asyncio.set_event_loop(loop)
+
+
+ .. availability:: Unix, Windows.
+
+
+.. class:: ProactorEventLoop
+
+ An event loop for Windows that uses "I/O Completion Ports" (IOCP).
+
+ .. availability:: Windows.
+
+ An example how to use :class:`ProactorEventLoop` on Windows::
+
+ import asyncio
+ import sys
+
+ if sys.platform == 'win32':
+ loop = asyncio.ProactorEventLoop()
+ asyncio.set_event_loop(loop)
+
+ .. seealso::
+
+ `MSDN documentation on I/O Completion Ports
+ `_.
+
+
+.. class:: AbstractEventLoop
+
+ Abstract base class for asyncio-compliant event loops.
+
+ The :ref:`Event Loop Methods ` section lists all
+ methods that an alternative implementation of ``AbstractEventLoop``
+ should have defined.
+
+
+Examples
+========
+
+Note that all examples in this section **purposefully** show how
+to use the low-level event loop APIs, such as :meth:`loop.run_forever`
+and :meth:`loop.call_soon`. Modern asyncio applications rarely
+need to be written this way; consider using the high-level functions
+like :func:`asyncio.run`.
+
+
+.. _asyncio_example_lowlevel_helloworld:
Hello World with call_soon()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Example using the :meth:`AbstractEventLoop.call_soon` method to schedule a
-callback. The callback displays ``"Hello World"`` and then stops the event
-loop::
+An example using the :meth:`loop.call_soon` method to schedule a
+callback. The callback displays ``"Hello World"`` and then stops the
+event loop::
import asyncio
def hello_world(loop):
+ """A callback to print 'Hello World' and stop the event loop"""
print('Hello World')
loop.stop()
@@ -882,23 +1482,25 @@ loop::
loop.call_soon(hello_world, loop)
# Blocking call interrupted by loop.stop()
- loop.run_forever()
- loop.close()
+ try:
+ loop.run_forever()
+ finally:
+ loop.close()
.. seealso::
- The :ref:`Hello World coroutine ` example
- uses a :ref:`coroutine `.
+ A similar :ref:`Hello World `
+ example created with a coroutine and the :func:`run` function.
-.. _asyncio-date-callback:
+.. _asyncio_example_call_later:
Display the current date with call_later()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Example of callback displaying the current date every second. The callback uses
-the :meth:`AbstractEventLoop.call_later` method to reschedule itself during 5
-seconds, and then stops the event loop::
+An example of a callback displaying the current date every second. The
+callback uses the :meth:`loop.call_later` method to reschedule itself
+after 5 seconds, and then stops the event loop::
import asyncio
import datetime
@@ -917,39 +1519,40 @@ seconds, and then stops the event loop::
loop.call_soon(display_date, end_time, loop)
# Blocking call interrupted by loop.stop()
- loop.run_forever()
- loop.close()
+ try:
+ loop.run_forever()
+ finally:
+ loop.close()
.. seealso::
- The :ref:`coroutine displaying the current date
- ` example uses a :ref:`coroutine
- `.
+ A similar :ref:`current date ` example
+ created with a coroutine and the :func:`run` function.
-.. _asyncio-watch-read-event:
+.. _asyncio_example_watch_fd:
Watch a file descriptor for read events
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Wait until a file descriptor received some data using the
-:meth:`AbstractEventLoop.add_reader` method and then close the event loop::
+:meth:`loop.add_reader` method and then close the event loop::
import asyncio
- try:
- from socket import socketpair
- except ImportError:
- from asyncio.windows_utils import socketpair
+ from socket import socketpair
# Create a pair of connected file descriptors
rsock, wsock = socketpair()
+
loop = asyncio.get_event_loop()
def reader():
data = rsock.recv(100)
print("Received:", data.decode())
+
# We are done: unregister the file descriptor
loop.remove_reader(rsock)
+
# Stop the event loop
loop.stop()
@@ -959,30 +1562,35 @@ Wait until a file descriptor received some data using the
# Simulate the reception of data from the network
loop.call_soon(wsock.send, 'abc'.encode())
- # Run the event loop
- loop.run_forever()
-
- # We are done, close sockets and the event loop
- rsock.close()
- wsock.close()
- loop.close()
+ try:
+ # Run the event loop
+ loop.run_forever()
+ finally:
+ # We are done. Close sockets and the event loop.
+ rsock.close()
+ wsock.close()
+ loop.close()
.. seealso::
- The :ref:`register an open socket to wait for data using a protocol
- ` example uses a low-level protocol created by the
- :meth:`AbstractEventLoop.create_connection` method.
+ * A similar :ref:`example `
+ using transports, protocols, and the
+ :meth:`loop.create_connection` method.
+
+ * Another similar :ref:`example `
+ using the high-level :func:`asyncio.open_connection` function
+ and streams.
- The :ref:`register an open socket to wait for data using streams
- ` example uses high-level streams
- created by the :func:`open_connection` function in a coroutine.
+.. _asyncio_example_unix_signals:
Set signal handlers for SIGINT and SIGTERM
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Register handlers for signals :py:data:`SIGINT` and :py:data:`SIGTERM` using
-the :meth:`AbstractEventLoop.add_signal_handler` method::
+(This ``signals`` example only works on Unix.)
+
+Register handlers for signals :py:data:`SIGINT` and :py:data:`SIGTERM`
+using the :meth:`loop.add_signal_handler` method::
import asyncio
import functools
@@ -993,16 +1601,17 @@ the :meth:`AbstractEventLoop.add_signal_handler` method::
print("got signal %s: exit" % signame)
loop.stop()
- loop = asyncio.get_event_loop()
- for signame in ('SIGINT', 'SIGTERM'):
- loop.add_signal_handler(getattr(signal, signame),
- functools.partial(ask_exit, signame))
+ async def main():
+ loop = asyncio.get_running_loop()
- print("Event loop running forever, press Ctrl+C to interrupt.")
- print("pid %s: send SIGINT or SIGTERM to exit." % os.getpid())
- try:
- loop.run_forever()
- finally:
- loop.close()
+ for signame in {'SIGINT', 'SIGTERM'}:
+ loop.add_signal_handler(
+ getattr(signal, signame),
+ functools.partial(ask_exit, signame))
+
+ await asyncio.sleep(3600)
+
+ print("Event loop running for 1 hour, press Ctrl+C to interrupt.")
+ print(f"pid {os.getpid()}: send SIGINT or SIGTERM to exit.")
-This example only works on UNIX.
+ asyncio.run(main())
diff --git a/Doc/library/asyncio-eventloops.rst b/Doc/library/asyncio-eventloops.rst
deleted file mode 100644
index 3aa9ffeff268c429473b018f4e5163a8a4ebbdfb..0000000000000000000000000000000000000000
--- a/Doc/library/asyncio-eventloops.rst
+++ /dev/null
@@ -1,232 +0,0 @@
-.. currentmodule:: asyncio
-
-Event loops
-===========
-
-**Source code:** :source:`Lib/asyncio/events.py`
-
-Event loop functions
---------------------
-
-The following functions are convenient shortcuts to accessing the methods of the
-global policy. Note that this provides access to the default policy, unless an
-alternative policy was set by calling :func:`set_event_loop_policy` earlier in
-the execution of the process.
-
-.. function:: get_event_loop()
-
- Equivalent to calling ``get_event_loop_policy().get_event_loop()``.
-
-.. function:: set_event_loop(loop)
-
- Equivalent to calling ``get_event_loop_policy().set_event_loop(loop)``.
-
-.. function:: new_event_loop()
-
- Equivalent to calling ``get_event_loop_policy().new_event_loop()``.
-
-
-.. _asyncio-event-loops:
-
-Available event loops
----------------------
-
-asyncio currently provides two implementations of event loops:
-:class:`SelectorEventLoop` and :class:`ProactorEventLoop`.
-
-.. class:: SelectorEventLoop
-
- Event loop based on the :mod:`selectors` module. Subclass of
- :class:`AbstractEventLoop`.
-
- Use the most efficient selector available on the platform.
-
- On Windows, only sockets are supported (ex: pipes are not supported):
- see the `MSDN documentation of select
- `_.
-
-.. class:: ProactorEventLoop
-
- Proactor event loop for Windows using "I/O Completion Ports" aka IOCP.
- Subclass of :class:`AbstractEventLoop`.
-
- Availability: Windows.
-
- .. seealso::
-
- `MSDN documentation on I/O Completion Ports
- `_.
-
-Example to use a :class:`ProactorEventLoop` on Windows::
-
- import asyncio, sys
-
- if sys.platform == 'win32':
- loop = asyncio.ProactorEventLoop()
- asyncio.set_event_loop(loop)
-
-.. _asyncio-platform-support:
-
-Platform support
-----------------
-
-The :mod:`asyncio` module has been designed to be portable, but each platform
-still has subtle differences and may not support all :mod:`asyncio` features.
-
-Windows
-^^^^^^^
-
-Common limits of Windows event loops:
-
-- :meth:`~AbstractEventLoop.create_unix_connection` and
- :meth:`~AbstractEventLoop.create_unix_server` are not supported: the socket
- family :data:`socket.AF_UNIX` is specific to UNIX
-- :meth:`~AbstractEventLoop.add_signal_handler` and
- :meth:`~AbstractEventLoop.remove_signal_handler` are not supported
-- :meth:`EventLoopPolicy.set_child_watcher` is not supported.
- :class:`ProactorEventLoop` supports subprocesses. It has only one
- implementation to watch child processes, there is no need to configure it.
-
-:class:`SelectorEventLoop` specific limits:
-
-- :class:`~selectors.SelectSelector` is used which only supports sockets
- and is limited to 512 sockets.
-- :meth:`~AbstractEventLoop.add_reader` and :meth:`~AbstractEventLoop.add_writer` only
- accept file descriptors of sockets
-- Pipes are not supported
- (ex: :meth:`~AbstractEventLoop.connect_read_pipe`,
- :meth:`~AbstractEventLoop.connect_write_pipe`)
-- :ref:`Subprocesses ` are not supported
- (ex: :meth:`~AbstractEventLoop.subprocess_exec`,
- :meth:`~AbstractEventLoop.subprocess_shell`)
-
-:class:`ProactorEventLoop` specific limits:
-
-- :meth:`~AbstractEventLoop.create_datagram_endpoint` (UDP) is not supported
-- :meth:`~AbstractEventLoop.add_reader` and :meth:`~AbstractEventLoop.add_writer` are
- not supported
-
-The resolution of the monotonic clock on Windows is usually around 15.6 msec.
-The best resolution is 0.5 msec. The resolution depends on the hardware
-(availability of `HPET
-`_) and on the Windows
-configuration. See :ref:`asyncio delayed calls `.
-
-.. versionchanged:: 3.5
-
- :class:`ProactorEventLoop` now supports SSL.
-
-
-Mac OS X
-^^^^^^^^
-
-Character devices like PTY are only well supported since Mavericks (Mac OS
-10.9). They are not supported at all on Mac OS 10.5 and older.
-
-On Mac OS 10.6, 10.7 and 10.8, the default event loop is
-:class:`SelectorEventLoop` which uses :class:`selectors.KqueueSelector`.
-:class:`selectors.KqueueSelector` does not support character devices on these
-versions. The :class:`SelectorEventLoop` can be used with
-:class:`~selectors.SelectSelector` or :class:`~selectors.PollSelector` to
-support character devices on these versions of Mac OS X. Example::
-
- import asyncio
- import selectors
-
- selector = selectors.SelectSelector()
- loop = asyncio.SelectorEventLoop(selector)
- asyncio.set_event_loop(loop)
-
-
-Event loop policies and the default policy
-------------------------------------------
-
-Event loop management is abstracted with a *policy* pattern, to provide maximal
-flexibility for custom platforms and frameworks. Throughout the execution of a
-process, a single global policy object manages the event loops available to the
-process based on the calling context. A policy is an object implementing the
-:class:`AbstractEventLoopPolicy` interface.
-
-For most users of :mod:`asyncio`, policies never have to be dealt with
-explicitly, since the default global policy is sufficient (see below).
-
-The module-level functions
-:func:`get_event_loop` and :func:`set_event_loop` provide convenient access to
-event loops managed by the default policy.
-
-
-Event loop policy interface
----------------------------
-
-An event loop policy must implement the following interface:
-
-.. class:: AbstractEventLoopPolicy
-
- Event loop policy.
-
- .. method:: get_event_loop()
-
- Get the event loop for the current context.
-
- Returns an event loop object implementing the :class:`AbstractEventLoop`
- interface. In case called from coroutine, it returns the currently
- running event loop.
-
- Raises an exception in case no event loop has been set for the current
- context and the current policy does not specify to create one. It must
- never return ``None``.
-
- .. versionchanged:: 3.6
-
- .. method:: set_event_loop(loop)
-
- Set the event loop for the current context to *loop*.
-
- .. method:: new_event_loop()
-
- Create and return a new event loop object according to this policy's
- rules.
-
- If there's need to set this loop as the event loop for the current
- context, :meth:`set_event_loop` must be called explicitly.
-
-
-The default policy defines context as the current thread, and manages an event
-loop per thread that interacts with :mod:`asyncio`. If the current thread
-doesn't already have an event loop associated with it, the default policy's
-:meth:`~AbstractEventLoopPolicy.get_event_loop` method creates one when
-called from the main thread, but raises :exc:`RuntimeError` otherwise.
-
-
-Access to the global loop policy
---------------------------------
-
-.. function:: get_event_loop_policy()
-
- Get the current event loop policy.
-
-.. function:: set_event_loop_policy(policy)
-
- Set the current event loop policy. If *policy* is ``None``, the default
- policy is restored.
-
-
-Customizing the event loop policy
----------------------------------
-
-To implement a new event loop policy, it is recommended you subclass the
-concrete default event loop policy :class:`DefaultEventLoopPolicy`
-and override the methods for which you want to change behavior, for example::
-
- class MyEventLoopPolicy(asyncio.DefaultEventLoopPolicy):
-
- def get_event_loop(self):
- """Get the event loop.
-
- This may be None or an instance of EventLoop.
- """
- loop = super().get_event_loop()
- # Do something with loop ...
- return loop
-
- asyncio.set_event_loop_policy(MyEventLoopPolicy())
diff --git a/Doc/library/asyncio-exceptions.rst b/Doc/library/asyncio-exceptions.rst
new file mode 100644
index 0000000000000000000000000000000000000000..dbd5df720850bdba1c2b7a73b7cf0cb806e380be
--- /dev/null
+++ b/Doc/library/asyncio-exceptions.rst
@@ -0,0 +1,91 @@
+.. currentmodule:: asyncio
+
+
+.. _asyncio-exceptions:
+
+==========
+Exceptions
+==========
+
+
+.. exception:: TimeoutError
+
+ The operation has exceeded the given deadline.
+
+ .. important::
+ This exception is different from the builtin :exc:`TimeoutError`
+ exception.
+
+
+.. exception:: CancelledError
+
+ The operation has been cancelled.
+
+ This exception can be caught to perform custom operations
+ when asyncio Tasks are cancelled. In almost all situations the
+ exception must be re-raised.
+
+ .. important::
+
+ This exception is a subclass of :exc:`Exception`, so it can be
+ accidentally suppressed by an overly broad ``try..except`` block::
+
+ try:
+ await operation
+ except Exception:
+ # The cancellation is broken because the *except* block
+ # suppresses the CancelledError exception.
+ log.log('an error has occurred')
+
+ Instead, the following pattern should be used::
+
+ try:
+ await operation
+ except asyncio.CancelledError:
+ raise
+ except Exception:
+ log.log('an error has occurred')
+
+
+.. exception:: InvalidStateError
+
+ Invalid internal state of :class:`Task` or :class:`Future`.
+
+ Can be raised in situations like setting a result value for a
+ *Future* object that already has a result value set.
+
+
+.. exception:: SendfileNotAvailableError
+
+ The "sendfile" syscall is not available for the given
+ socket or file type.
+
+ A subclass of :exc:`RuntimeError`.
+
+
+.. exception:: IncompleteReadError
+
+ The requested read operation did not complete fully.
+
+ Raised by the :ref:`asyncio stream APIs`.
+
+ This exception is a subclass of :exc:`EOFError`.
+
+ .. attribute:: expected
+
+ The total number (:class:`int`) of expected bytes.
+
+ .. attribute:: partial
+
+ A string of :class:`bytes` read before the end of stream was reached.
+
+
+.. exception:: LimitOverrunError
+
+ Reached the buffer size limit while looking for a separator.
+
+ Raised by the :ref:`asyncio stream APIs `.
+
+ .. attribute:: consumed
+
+ The total number of to be consumed bytes.
diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst
new file mode 100644
index 0000000000000000000000000000000000000000..6e6e0137c1bdda4de224a63dc6a7c27fda4b4a34
--- /dev/null
+++ b/Doc/library/asyncio-future.rst
@@ -0,0 +1,250 @@
+.. currentmodule:: asyncio
+
+
+.. _asyncio-futures:
+
+=======
+Futures
+=======
+
+*Future* objects are used to bridge **low-level callback-based code**
+with high-level async/await code.
+
+
+Future Functions
+================
+
+.. function:: isfuture(obj)
+
+ Return ``True`` if *obj* is either of:
+
+ * an instance of :class:`asyncio.Future`,
+ * an instance of :class:`asyncio.Task`,
+ * a Future-like object with a ``_asyncio_future_blocking``
+ attribute.
+
+ .. versionadded:: 3.5
+
+
+.. function:: ensure_future(obj, \*, loop=None)
+
+ Return:
+
+ * *obj* argument as is, if *obj* is a :class:`Future`,
+ a :class:`Task`, or a Future-like object (:func:`isfuture`
+ is used for the test.)
+
+ * a :class:`Task` object wrapping *obj*, if *obj* is a
+ coroutine (:func:`iscoroutine` is used for the test.)
+
+ * a :class:`Task` object that would await on *obj*, if *obj* is an
+ awaitable (:func:`inspect.isawaitable` is used for the test.)
+
+ If *obj* is neither of the above a :exc:`TypeError` is raised.
+
+ .. important::
+
+ See also the :func:`create_task` function which is the
+ preferred way for creating new Tasks.
+
+ .. versionchanged:: 3.5.1
+ The function accepts any :term:`awaitable` object.
+
+
+.. function:: wrap_future(future, \*, loop=None)
+
+ Wrap a :class:`concurrent.futures.Future` object in a
+ :class:`asyncio.Future` object.
+
+
+Future Object
+=============
+
+.. class:: Future(\*, loop=None)
+
+ A Future represents an eventual result of an asynchronous
+ operation. Not thread-safe.
+
+ Future is an :term:`awaitable` object. Coroutines can await on
+ Future objects until they either have a result or an exception
+ set, or until they are cancelled.
+
+ Typically Futures are used to enable low-level
+ callback-based code (e.g. in protocols implemented using asyncio
+ :ref:`transports `)
+ to interoperate with high-level async/await code.
+
+ The rule of thumb is to never expose Future objects in user-facing
+ APIs, and the recommended way to create a Future object is to call
+ :meth:`loop.create_future`. This way alternative event loop
+ implementations can inject their own optimized implementations
+ of a Future object.
+
+ .. versionchanged:: 3.7
+ Added support for the :mod:`contextvars` module.
+
+ .. method:: result()
+
+ Return the result of the Future.
+
+ If the Future is *done* and has a result set by the
+ :meth:`set_result` method, the result value is returned.
+
+ If the Future is *done* and has an exception set by the
+ :meth:`set_exception` method, this method raises the exception.
+
+ If the Future has been *cancelled*, this method raises
+ a :exc:`CancelledError` exception.
+
+ If the Future's result isn't yet available, this method raises
+ a :exc:`InvalidStateError` exception.
+
+ .. method:: set_result(result)
+
+ Mark the Future as *done* and set its result.
+
+ Raises a :exc:`InvalidStateError` error if the Future is
+ already *done*.
+
+ .. method:: set_exception(exception)
+
+ Mark the Future as *done* and set an exception.
+
+ Raises a :exc:`InvalidStateError` error if the Future is
+ already *done*.
+
+ .. method:: done()
+
+ Return ``True`` if the Future is *done*.
+
+ A Future is *done* if it was *cancelled* or if it has a result
+ or an exception set with :meth:`set_result` or
+ :meth:`set_exception` calls.
+
+ .. method:: cancelled()
+
+ Return ``True`` if the Future was *cancelled*.
+
+ The method is usually used to check if a Future is not
+ *cancelled* before setting a result or an exception for it::
+
+ if not fut.cancelled():
+ fut.set_result(42)
+
+ .. method:: add_done_callback(callback, *, context=None)
+
+ Add a callback to be run when the Future is *done*.
+
+ The *callback* is called with the Future object as its only
+ argument.
+
+ If the Future is already *done* when this method is called,
+ the callback is scheduled with :meth:`loop.call_soon`.
+
+ An optional keyword-only *context* argument allows specifying a
+ custom :class:`contextvars.Context` for the *callback* to run in.
+ The current context is used when no *context* is provided.
+
+ :func:`functools.partial` can be used to pass parameters
+ to the callback, e.g.::
+
+ # Call 'print("Future:", fut)' when "fut" is done.
+ fut.add_done_callback(
+ functools.partial(print, "Future:"))
+
+ .. versionchanged:: 3.7
+ The *context* keyword-only parameter was added.
+ See :pep:`567` for more details.
+
+ .. method:: remove_done_callback(callback)
+
+ Remove *callback* from the callbacks list.
+
+ Returns the number of callbacks removed, which is typically 1,
+ unless a callback was added more than once.
+
+ .. method:: cancel()
+
+ Cancel the Future and schedule callbacks.
+
+ If the Future is already *done* or *cancelled*, return ``False``.
+ Otherwise, change the Future's state to *cancelled*,
+ schedule the callbacks, and return ``True``.
+
+ .. method:: exception()
+
+ Return the exception that was set on this Future.
+
+ The exception (or ``None`` if no exception was set) is
+ returned only if the Future is *done*.
+
+ If the Future has been *cancelled*, this method raises a
+ :exc:`CancelledError` exception.
+
+ If the Future isn't *done* yet, this method raises an
+ :exc:`InvalidStateError` exception.
+
+ .. method:: get_loop()
+
+ Return the event loop the Future object is bound to.
+
+ .. versionadded:: 3.7
+
+
+.. _asyncio_example_future:
+
+This example creates a Future object, creates and schedules an
+asynchronous Task to set result for the Future, and waits until
+the Future has a result::
+
+ async def set_after(fut, delay, value):
+ # Sleep for *delay* seconds.
+ await asyncio.sleep(delay)
+
+ # Set *value* as a result of *fut* Future.
+ fut.set_result(value)
+
+ async def main():
+ # Get the current event loop.
+ loop = asyncio.get_running_loop()
+
+ # Create a new Future object.
+ fut = loop.create_future()
+
+ # Run "set_after()" coroutine in a parallel Task.
+ # We are using the low-level "loop.create_task()" API here because
+ # we already have a reference to the event loop at hand.
+ # Otherwise we could have just used "asyncio.create_task()".
+ loop.create_task(
+ set_after(fut, 1, '... world'))
+
+ print('hello ...')
+
+ # Wait until *fut* has a result (1 second) and print it.
+ print(await fut)
+
+ asyncio.run(main())
+
+
+.. important::
+
+ The Future object was designed to mimic
+ :class:`concurrent.futures.Future`. Key differences include:
+
+ - unlike asyncio Futures, :class:`concurrent.futures.Future`
+ instances cannot be awaited.
+
+ - :meth:`asyncio.Future.result` and :meth:`asyncio.Future.exception`
+ do not accept the *timeout* argument.
+
+ - :meth:`asyncio.Future.result` and :meth:`asyncio.Future.exception`
+ raise an :exc:`InvalidStateError` exception when the Future is not
+ *done*.
+
+ - Callbacks registered with :meth:`asyncio.Future.add_done_callback`
+ are not called immediately. They are scheduled with
+ :meth:`loop.call_soon` instead.
+
+ - asyncio Future is not compatible with the
+ :func:`concurrent.futures.wait` and
+ :func:`concurrent.futures.as_completed` functions.
diff --git a/Doc/library/asyncio-llapi-index.rst b/Doc/library/asyncio-llapi-index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..0ab322af6dc72da6ccf63da0c10927821029a823
--- /dev/null
+++ b/Doc/library/asyncio-llapi-index.rst
@@ -0,0 +1,510 @@
+.. currentmodule:: asyncio
+
+
+===================
+Low-level API Index
+===================
+
+This page lists all low-level asyncio APIs.
+
+
+Obtaining the Event Loop
+========================
+
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - :func:`asyncio.get_running_loop`
+ - The **preferred** function to get the running event loop.
+
+ * - :func:`asyncio.get_event_loop`
+ - Get an event loop instance (current or via the policy).
+
+ * - :func:`asyncio.set_event_loop`
+ - Set the event loop as current via the current policy.
+
+ * - :func:`asyncio.new_event_loop`
+ - Create a new event loop.
+
+
+.. rubric:: Examples
+
+* :ref:`Using asyncio.get_running_loop() `.
+
+
+Event Loop Methods
+==================
+
+See also the main documentation section about the
+:ref:`event loop methods `.
+
+.. rubric:: Lifecycle
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - :meth:`loop.run_until_complete`
+ - Run a Future/Task/awaitable until complete.
+
+ * - :meth:`loop.run_forever`
+ - Run the event loop forever.
+
+ * - :meth:`loop.stop`
+ - Stop the event loop.
+
+ * - :meth:`loop.close`
+ - Close the event loop.
+
+ * - :meth:`loop.is_running()`
+ - Return ``True`` if the event loop is running.
+
+ * - :meth:`loop.is_closed()`
+ - Return ``True`` if the event loop is closed.
+
+ * - ``await`` :meth:`loop.shutdown_asyncgens`
+ - Close asynchronous generators.
+
+
+.. rubric:: Debugging
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - :meth:`loop.set_debug`
+ - Enable or disable the debug mode.
+
+ * - :meth:`loop.get_debug`
+ - Get the current debug mode.
+
+
+.. rubric:: Scheduling Callbacks
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - :meth:`loop.call_soon`
+ - Invoke a callback soon.
+
+ * - :meth:`loop.call_soon_threadsafe`
+ - A thread-safe variant of :meth:`loop.call_soon`.
+
+ * - :meth:`loop.call_later`
+ - Invoke a callback *after* the given time.
+
+ * - :meth:`loop.call_at`
+ - Invoke a callback *at* the given time.
+
+
+.. rubric:: Thread/Process Pool
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - ``await`` :meth:`loop.run_in_executor`
+ - Run a CPU-bound or other blocking function in
+ a :mod:`concurrent.futures` executor.
+
+ * - :meth:`loop.set_default_executor`
+ - Set the default executor for :meth:`loop.run_in_executor`.
+
+
+.. rubric:: Tasks and Futures
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - :meth:`loop.create_future`
+ - Create a :class:`Future` object.
+
+ * - :meth:`loop.create_task`
+ - Schedule coroutine as a :class:`Task`.
+
+ * - :meth:`loop.set_task_factory`
+ - Set a factory used by :meth:`loop.create_task` to
+ create :class:`Tasks `.
+
+ * - :meth:`loop.get_task_factory`
+ - Get the factory :meth:`loop.create_task` uses
+ to create :class:`Tasks `.
+
+
+.. rubric:: DNS
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - ``await`` :meth:`loop.getaddrinfo`
+ - Asynchronous version of :meth:`socket.getaddrinfo`.
+
+ * - ``await`` :meth:`loop.getnameinfo`
+ - Asynchronous version of :meth:`socket.getnameinfo`.
+
+
+.. rubric:: Networking and IPC
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - ``await`` :meth:`loop.create_connection`
+ - Open a TCP connection.
+
+ * - ``await`` :meth:`loop.create_server`
+ - Create a TCP server.
+
+ * - ``await`` :meth:`loop.create_unix_connection`
+ - Open a Unix socket connection.
+
+ * - ``await`` :meth:`loop.create_unix_server`
+ - Create a Unix socket server.
+
+ * - ``await`` :meth:`loop.connect_accepted_socket`
+ - Wrap a :class:`~socket.socket` into a ``(transport, protocol)``
+ pair.
+
+ * - ``await`` :meth:`loop.create_datagram_endpoint`
+ - Open a datagram (UDP) connection.
+
+ * - ``await`` :meth:`loop.sendfile`
+ - Send a file over a transport.
+
+ * - ``await`` :meth:`loop.start_tls`
+ - Upgrade an existing connection to TLS.
+
+ * - ``await`` :meth:`loop.connect_read_pipe`
+ - Wrap a read end of a pipe into a ``(transport, protocol)`` pair.
+
+ * - ``await`` :meth:`loop.connect_write_pipe`
+ - Wrap a write end of a pipe into a ``(transport, protocol)`` pair.
+
+
+.. rubric:: Sockets
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - ``await`` :meth:`loop.sock_recv`
+ - Receive data from the :class:`~socket.socket`.
+
+ * - ``await`` :meth:`loop.sock_recv_into`
+ - Receive data from the :class:`~socket.socket` into a buffer.
+
+ * - ``await`` :meth:`loop.sock_sendall`
+ - Send data to the :class:`~socket.socket`.
+
+ * - ``await`` :meth:`loop.sock_connect`
+ - Connect the :class:`~socket.socket`.
+
+ * - ``await`` :meth:`loop.sock_accept`
+ - Accept a :class:`~socket.socket` connection.
+
+ * - ``await`` :meth:`loop.sock_sendfile`
+ - Send a file over the :class:`~socket.socket`.
+
+ * - :meth:`loop.add_reader`
+ - Start watching a file descriptor for read availability.
+
+ * - :meth:`loop.remove_reader`
+ - Stop watching a file descriptor for read availability.
+
+ * - :meth:`loop.add_writer`
+ - Start watching a file descriptor for write availability.
+
+ * - :meth:`loop.remove_writer`
+ - Stop watching a file descriptor for write availability.
+
+
+.. rubric:: Unix Signals
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - :meth:`loop.add_signal_handler`
+ - Add a handler for a :mod:`signal`.
+
+ * - :meth:`loop.remove_signal_handler`
+ - Remove a handler for a :mod:`signal`.
+
+
+.. rubric:: Subprocesses
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - :meth:`loop.subprocess_exec`
+ - Spawn a subprocess.
+
+ * - :meth:`loop.subprocess_shell`
+ - Spawn a subprocess from a shell command.
+
+
+.. rubric:: Error Handling
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - :meth:`loop.call_exception_handler`
+ - Call the exception handler.
+
+ * - :meth:`loop.set_exception_handler`
+ - Set a new exception handler.
+
+ * - :meth:`loop.get_exception_handler`
+ - Get the current exception handler.
+
+ * - :meth:`loop.default_exception_handler`
+ - The default exception handler implementation.
+
+
+.. rubric:: Examples
+
+* :ref:`Using asyncio.get_event_loop() and loop.run_forever()
+ `.
+
+* :ref:`Using loop.call_later() `.
+
+* Using ``loop.create_connection()`` to implement
+ :ref:`an echo-client `.
+
+* Using ``loop.create_connection()`` to
+ :ref:`connect a socket `.
+
+* :ref:`Using add_reader() to watch an FD for read events
+ `.
+
+* :ref:`Using loop.add_signal_handler() `.
+
+* :ref:`Using loop.subprocess_exec() `.
+
+
+Transports
+==========
+
+All transports implement the following methods:
+
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - :meth:`transport.close() `
+ - Close the transport.
+
+ * - :meth:`transport.is_closing() `
+ - Return ``True`` if the transport is closing or is closed.
+
+ * - :meth:`transport.get_extra_info() `
+ - Request for information about the transport.
+
+ * - :meth:`transport.set_protocol() `
+ - Set a new protocol.
+
+ * - :meth:`transport.get_protocol() `
+ - Return the current protocol.
+
+
+Transports that can receive data (TCP and Unix connections,
+pipes, etc). Returned from methods like
+:meth:`loop.create_connection`, :meth:`loop.create_unix_connection`,
+:meth:`loop.connect_read_pipe`, etc:
+
+.. rubric:: Read Transports
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - :meth:`transport.is_reading() `
+ - Return ``True`` if the transport is receiving.
+
+ * - :meth:`transport.pause_reading() `
+ - Pause receiving.
+
+ * - :meth:`transport.resume_reading() `
+ - Resume receiving.
+
+
+Transports that can Send data (TCP and Unix connections,
+pipes, etc). Returned from methods like
+:meth:`loop.create_connection`, :meth:`loop.create_unix_connection`,
+:meth:`loop.connect_write_pipe`, etc:
+
+.. rubric:: Write Transports
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - :meth:`transport.write() `
+ - Write data to the transport.
+
+ * - :meth:`transport.writelines() `
+ - Write buffers to the transport.
+
+ * - :meth:`transport.can_write_eof() `
+ - Return :const:`True` if the transport supports sending EOF.
+
+ * - :meth:`transport.write_eof() `
+ - Close and send EOF after flushing buffered data.
+
+ * - :meth:`transport.abort() `
+ - Close the transport immediately.
+
+ * - :meth:`transport.get_write_buffer_size()
+ `
+ - Return high and low water marks for write flow control.
+
+ * - :meth:`transport.set_write_buffer_limits()
+ `
+ - Set new high and low water marks for write flow control.
+
+
+Transports returned by :meth:`loop.create_datagram_endpoint`:
+
+.. rubric:: Datagram Transports
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - :meth:`transport.sendto() `
+ - Send data to the remote peer.
+
+ * - :meth:`transport.abort() `
+ - Close the transport immediately.
+
+
+Low-level transport abstraction over subprocesses.
+Returned by :meth:`loop.subprocess_exec` and
+:meth:`loop.subprocess_shell`:
+
+.. rubric:: Subprocess Transports
+.. list-table::
+ :widths: 50 50
+ :class: full-width-table
+
+ * - :meth:`transport.get_pid() `
+ - Return the subprocess process id.
+
+ * - :meth:`transport.get_pipe_transport()
+ `
+ - Return the transport for the requested communication pipe
+ (*stdin*, *stdout*, or *stderr*).
+
+ * - :meth:`transport.get_returncode() `
+ - Return the subprocess return code.
+
+ * - :meth:`transport.kill()