diff --git a/.gitignore b/.gitignore index c227f50d55299e873fe199f12234195db7cff3eb..be75938ec401b1d72fa54773c85191aaac7d7f35 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ node_modules /bazel-* /bazel_pip /tools/python_bin_path.sh -/tools/git/gen +/tensorflow/tools/git/gen /pip_test /_python_build *.pyc @@ -15,3 +15,22 @@ __pycache__ .vscode/ cmake_build/ .idea/** +/build/ +/tensorflow/core/util/version_info.cc +/tensorflow/python/framework/fast_tensor_util.cpp +Pods +Podfile.lock +*.pbxproj +*.xcworkspacedata +/tensorflow/contrib/lite/downloads/** +/tensorflow/contrib/lite/gen/** +/tensorflow/contrib/lite/examples/ios/simple/data/*.txt +/tensorflow/contrib/lite/examples/ios/simple/data/*.tflite +xcuserdata/** + +# Android +.gradle +.idea +*.iml +local.properties +gradleBuild diff --git a/AUTHORS b/AUTHORS index a46ae7e616ab3a420d9fb2691ee8d8650032a39f..aa4be5169dcc68c579863e8ba6307cd00e9f9a68 100644 --- a/AUTHORS +++ b/AUTHORS @@ -7,4 +7,4 @@ # The email address is not required for organizations. Google Inc. -Yuan Tang terrytangyuan@gmail.com +Yuan Tang diff --git a/CODEOWNERS b/CODEOWNERS index 0a12176aaa27fb2909d623e3bced896b04d96584..007a304c3e706ce968576ec8979c08f1a3bcc552 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,53 +1,53 @@ # NOTE: Disabled temporarily because it's too noisy on pushes. # Where component owners are known, add them here. -#tensorflow/core/platform/windows/* @mrry -#tensorflow/java/* @asimshankar -#tensorflow/tensorboard/* @jart @dandelionmane -#tensorflow/tools/docs/* @markdaoust +# /tensorflow/core/platform/windows/ @mrry +# /tensorflow/java/ @asimshankar +# /tensorflow/tensorboard/ @jart @dandelionmane +# /tensorflow/tools/docs/ @markdaoust # contrib -# NEED OWNER: tensorflow/contrib/avro/* -#tensorflow/contrib/batching/* @alextp @chrisolston -#tensorflow/contrib/bayesflow/* @ebrevdo @rsepassi @jvdillon -#tensorflow/contrib/cmake/* @mrry @benoitsteiner -#tensorflow/contrib/copy_graph/* @tucker @poxvoculi -#tensorflow/contrib/crf/* @kentonl -#tensorflow/contrib/data/* @mrry -#tensorflow/contrib/distributions/* @jvdillon @langmore @rsepassi -#tensorflow/contrib/factorization/* @agarwal-ashish @xavigonzalvo -#tensorflow/contrib/ffmpeg/* @fredbertsch -# NEED OWNER: tensorflow/contrib/framework/* -#tensorflow/contrib/graph_editor/* @purpledog -# NEED OWNER: tensorflow/contrib/grid_rnn/* -#tensorflow/contrib/hvx/* @satok16 -#tensorflow/contrib/imperative/* @keveman -#tensorflow/contrib/integrate/* @shoyer -#tensorflow/contrib/kernel_methods/* @petrosmol -#tensorflow/contrib/ios_examples/* @petewarden -#tensorflow/contrib/labeled_tensor/* @shoyer -#tensorflow/contrib/layers/* @fchollet @martinwicke -#tensorflow/contrib/learn/* @martinwicke @ispirmustafa @alextp -#tensorflow/contrib/linalg/* @langmore -#tensorflow/contrib/linear_optimizer/* @petrosmol @andreasst @katsiapis -#tensorflow/contrib/lookup/* @ysuematsu @andreasst -#tensorflow/contrib/losses/* @alextp @ispirmustafa -#tensorflow/contrib/makefile/* @petewarden @satok16 @wolffg -#tensorflow/contrib/metrics/* @alextp @honkentuber @ispirmustafa -#tensorflow/contrib/nccl/* @cwhipkey @zheng-xq -#tensorflow/contrib/opt/* @strategist333 -#tensorflow/contrib/pi_examples/* @maciekcc -#tensorflow/contrib/quantization/* @petewarden @cwhipkey @keveman -#tensorflow/contrib/rnn/* @ebrevdo -#tensorflow/contrib/saved_model/* @nfiedel @sukritiramesh -#tensorflow/contrib/seq2seq/* @lukaszkaiser -#tensorflow/contrib/session_bundle/* @nfiedel @sukritiramesh -#tensorflow/contrib/slim/* @sguada @thenbasilmanran -#tensorflow/contrib/stateless/* @girving -#tensorflow/contrib/tensor_forest/* @gilberthendry @thomascolthurst -#tensorflow/contrib/testing/* @dandelionmane -#tensorflow/contrib/timeseries/* @allenlavoie -#tensorflow/contrib/tpu/* @frankchn @saeta @jhseu -#tensorflow/contrib/training/* @joel-shor @ebrevdo -#tensorflow/contrib/util/* @sherrym +# NEED OWNER: /tensorflow/contrib/avro/ +# /tensorflow/contrib/batching/ @alextp @chrisolston +# /tensorflow/contrib/bayesflow/ @ebrevdo @rsepassi @jvdillon +# /tensorflow/contrib/boosted_trees/ @sshrdp @yk5 @nataliaponomareva +# /tensorflow/contrib/cmake/ @mrry @benoitsteiner +# /tensorflow/contrib/copy_graph/ @tucker @poxvoculi +# /tensorflow/contrib/crf/ @kentonl +# /tensorflow/contrib/data/ @mrry +# /tensorflow/contrib/distributions/ @jvdillon @langmore @rsepassi +# /tensorflow/contrib/factorization/ @agarwal-ashish @xavigonzalvo +# /tensorflow/contrib/ffmpeg/ @fredbertsch +# NEED OWNER: /tensorflow/contrib/framework/ +# /tensorflow/contrib/graph_editor/ @purpledog +# NEED OWNER: /tensorflow/contrib/grid_rnn/ +# /tensorflow/contrib/hvx/ @satok16 +# /tensorflow/contrib/integrate/ @shoyer +# /tensorflow/contrib/kernel_methods/ @petrosmol +# /tensorflow/contrib/ios_examples/ @petewarden +# /tensorflow/contrib/labeled_tensor/ @shoyer +# /tensorflow/contrib/layers/ @fchollet @martinwicke +# /tensorflow/contrib/learn/ @martinwicke @ispirmustafa @alextp +# /tensorflow/contrib/linalg/ @langmore +# /tensorflow/contrib/linear_optimizer/ @petrosmol @andreasst @katsiapis +# /tensorflow/contrib/lookup/ @ysuematsu @andreasst +# /tensorflow/contrib/losses/ @alextp @ispirmustafa +# /tensorflow/contrib/makefile/ @petewarden @satok16 @wolffg +# /tensorflow/contrib/metrics/ @alextp @honkentuber @ispirmustafa +# /tensorflow/contrib/nccl/ @cwhipkey @zheng-xq +# /tensorflow/contrib/opt/ @strategist333 +# /tensorflow/contrib/pi_examples/ @maciekcc +# /tensorflow/contrib/quantization/ @petewarden @cwhipkey @keveman +# /tensorflow/contrib/rnn/ @ebrevdo +# /tensorflow/contrib/saved_model/ @nfiedel @sukritiramesh +# /tensorflow/contrib/seq2seq/ @lukaszkaiser +# /tensorflow/contrib/session_bundle/ @nfiedel @sukritiramesh +# /tensorflow/contrib/slim/ @sguada @thenbasilmanran +# /tensorflow/contrib/stateless/ @girving +# /tensorflow/contrib/tensor_forest/ @gilberthendry @thomascolthurst +# /tensorflow/contrib/testing/ @dandelionmane +# /tensorflow/contrib/timeseries/ @allenlavoie +# /tensorflow/contrib/tpu/ @frankchn @saeta @jhseu +# /tensorflow/contrib/training/ @joel-shor @ebrevdo +# /tensorflow/contrib/util/ @sherrym diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 10fd595fec7f240c3fdc871e1f32cc83f2ffd46d..5fff9d05a1c589636bc9c711e6eb7cc4aba86b2f 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -42,7 +42,7 @@ The Code of Conduct also applies within project spaces and in public spaces when Conflicts in an open source project can take many forms, from someone having a bad day and using harsh and hurtful language in the issue queue, to more serious instances such as sexist/racist statements or threats of violence, and everything in between. -If the behaviour is threatening or harassing, or for other reasons requires immediate escalation, please see below. +If the behavior is threatening or harassing, or for other reasons requires immediate escalation, please see below. However, for the vast majority of issues, we aim to empower individuals to first resolve conflicts themselves, asking for help when needed, and only after that fails to escalate further. This approach gives people more control over the outcome of their dispute. @@ -55,16 +55,16 @@ If you are experiencing or witnessing conflict, we ask you to use the following ## Reporting Violations -Violations of the Code of Conduct can be reported to TensorFlow’s Project Steward at conduct@tensorflow.org. The Project Steward will determine whether the Code of Conduct was violated, and will issue an appropriate sanction, possibly including a written warning or expulsion from the project, project sponsored spaces, or project forums. We ask that you make a good-faith effort to resolve your conflict via the conflict resolution policy before submitting a report. +Violations of the Code of Conduct can be reported to TensorFlow’s Project Stewards, Edd Wilder-James (ewj@google.com) and Sarah Novotny (sarahnovotny@google.com). The Project Steward will determine whether the Code of Conduct was violated, and will issue an appropriate sanction, possibly including a written warning or expulsion from the project, project sponsored spaces, or project forums. We ask that you make a good-faith effort to resolve your conflict via the conflict resolution policy before submitting a report. Violations of the Code of Conduct can occur in any setting, even those unrelated to the project. We will only consider complaints about conduct that has occurred within one year of the report. ## Enforcement -If the Project Steward receives a report alleging a violation of the Code of Conduct, the Project Steward will notify the accused of the report, and provide them an opportunity to discuss the report before a sanction is issued. The Project Steward will do their utmost to keep the reporter anonymous. If the act is ongoing (such as someone engaging in harassment), or involves a threat to anyone's safety (e.g. threats of violence), the Project Steward may issue sanctions without notice. +If the Project Stewards receive a report alleging a violation of the Code of Conduct, the Project Stewards will notify the accused of the report, and provide them an opportunity to discuss the report before a sanction is issued. The Project Stewards will do their utmost to keep the reporter anonymous. If the act is ongoing (such as someone engaging in harassment), or involves a threat to anyone's safety (e.g. threats of violence), the Project Stewards may issue sanctions without notice. ## Attribution -This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at http://contributor-covenant.org/version/1/4, and includes some aspects of the Geek Feminism Code of Conduct and the Drupal Code of Conduct. +This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://contributor-covenant.org/version/1/4, and includes some aspects of the Geek Feminism Code of Conduct and the Drupal Code of Conduct. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 43abdaafbf45379430920cd027b26299cd62553b..de4fded6ae6e66995aa9f1687a9d598017416f7a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,8 +8,8 @@ We'd love to accept your patches! Before we can take them, we have to jump a cou Please fill out either the individual or corporate Contributor License Agreement (CLA). - * If you are an individual writing original source code and you're sure you own the intellectual property, then you'll need to sign an [individual CLA](http://code.google.com/legal/individual-cla-v1.0.html). - * If you work for a company that wants to allow you to contribute your work, then you'll need to sign a [corporate CLA](http://code.google.com/legal/corporate-cla-v1.0.html). + * If you are an individual writing original source code and you're sure you own the intellectual property, then you'll need to sign an [individual CLA](https://code.google.com/legal/individual-cla-v1.0.html). + * If you work for a company that wants to allow you to contribute your work, then you'll need to sign a [corporate CLA](https://code.google.com/legal/corporate-cla-v1.0.html). Follow either of the two links above to access the appropriate CLA and instructions for how to sign and return it. Once we receive it, we'll be able to accept your pull requests. @@ -20,6 +20,9 @@ Follow either of the two links above to access the appropriate CLA and instructi If you have improvements to TensorFlow, send us your pull requests! For those just getting started, Github has a [howto](https://help.github.com/articles/using-pull-requests/). +TensorFlow team members will be assigned to review your pull requests. Once the pull requests are approved and pass continuous integration checks, we will merge the pull requests. +For some pull requests, we will apply the patch for each pull request to our internal version control system first, and export the change out as a new commit later, at which point the original pull request will be closed. The commits in the pull request will be squashed into a single commit with the pull request creator as the author. These pull requests will be labeled as pending merge internally. + If you want to contribute but you're not sure where to start, take a look at the [issues with the "contributions welcome" label](https://github.com/tensorflow/tensorflow/labels/stat%3Acontributions%20welcome). These are issues that we believe are particularly well suited for outside @@ -114,6 +117,7 @@ pylint --rcfile=/tmp/pylintrc myfile.py * [Google Java Style Guide](https://google.github.io/styleguide/javaguide.html) * [Google JavaScript Style Guide](https://google.github.io/styleguide/jsguide.html) * [Google Shell Style Guide](https://google.github.io/styleguide/shell.xml) +* [Google Objective-C Style Guide](https://google.github.io/styleguide/objcguide.html) #### Running sanity check diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md index 2bf2c754cf64ec3bac22a22fbafcebbd4dc54bf4..1a401997c649518766acb2ebb0dea1c128bd0ba4 100644 --- a/ISSUE_TEMPLATE.md +++ b/ISSUE_TEMPLATE.md @@ -19,6 +19,7 @@ If you open a GitHub issue, here is our policy: - **TensorFlow version (use command below)**: - **Python version**: - **Bazel version (if compiling from source)**: +- **GCC/Compiler version (if compiling from source)**: - **CUDA/cuDNN version**: - **GPU model and memory**: - **Exact command to reproduce**: diff --git a/README.md b/README.md index 87c7b1bfa9fb0021af6285ea4fd794658719d4a1..be5bda37451a1f0d678457e964420f1b75d0df16 100644 --- a/README.md +++ b/README.md @@ -36,13 +36,22 @@ and discussion, and please direct specific questions to [Stack Overflow](https:/ People who are a little more adventurous can also try our nightly binaries: - -* Linux CPU-only: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.3.0-cp27-none-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave)) / [Python 3.4](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.3.0-cp34-cp34m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/)) / [Python 3.5](https://ci.tensorflow.org/view/Nightly/job/nightly-python35-linux-cpu/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-python35-linux-cpu/)) -* Linux GPU: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow_gpu-1.3.0-cp27-none-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/)) / [Python 3.4](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow_gpu-1.3.0-cp34-cp34m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/)) / [Python 3.5](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow_gpu-1.3.0-cp35-cp35m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/)) -* Mac CPU-only: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.3.0-py2-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac-slave/)) / [Python 3](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.3.0-py3-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac-slave/)) -* Windows CPU-only: [Python 3.5 64-bit](https://ci.tensorflow.org/view/Nightly/job/nightly-win/M=windows,PY=35/lastSuccessfulBuild/artifact/cmake_build/tf_python/dist/tensorflow-1.3.0-cp35-cp35m-win_amd64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-win/M=windows,PY=35/)) / [Python 3.6 64-bit](https://ci.tensorflow.org/view/Nightly/job/nightly-win/M=windows,PY=36/lastSuccessfulBuild/artifact/cmake_build/tf_python/dist/tensorflow-1.3.0-cp36-cp36m-win_amd64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-win/M=windows,PY=36/)) -* Windows GPU: [Python 3.5 64-bit](https://ci.tensorflow.org/view/Nightly/job/nightly-win/M=windows-gpu,PY=35/lastSuccessfulBuild/artifact/cmake_build/tf_python/dist/tensorflow_gpu-1.3.0-cp35-cp35m-win_amd64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-win/M=windows-gpu,PY=35/)) / [Python 3.6 64-bit](https://ci.tensorflow.org/view/Nightly/job/nightly-win/M=windows-gpu,PY=36/lastSuccessfulBuild/artifact/cmake_build/tf_python/dist/tensorflow_gpu-1.3.0-cp36-cp36m-win_amd64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-win/M=windows-gpu,PY=36/)) -* Android: [demo APK](https://ci.tensorflow.org/view/Nightly/job/nightly-android/lastSuccessfulBuild/artifact/out/tensorflow_demo.apk), [native libs](http://ci.tensorflow.org/view/Nightly/job/nightly-android/lastSuccessfulBuild/artifact/out/native/) +**Nightly pip packages** +* We are pleased to announce that TensorFlow now offers nightly pip packages +under the [tf-nightly](https://pypi.python.org/pypi/tf-nightly) and +[tf-nightly-gpu](https://pypi.python.org/pypi/tf-nightly-gpu) project on pypi. +Simply run `pip install tf-nightly` or `pip install tf-nightly-gpu` in a clean +environment to install the nightly TensorFlow build. We support CPU and GPU +packages on Linux, Mac, and Windows. + + +**Individual whl files** +* Linux CPU-only: [Python 2](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tf_nightly-1.head-cp27-none-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/)) / [Python 3.4](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tf_nightly-1.head-cp34-cp34m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/)) / [Python 3.5](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tf_nightly-1.head-cp35-cp35m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=cpu-slave/)) / [Python 3.6](http://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.6,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tf_nightly-1.head-cp36-cp36m-linux_x86_64.whl) +* Linux GPU: [Python 2](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/42/artifact/pip_test/whl/tf_nightly_gpu-1.head-cp27-none-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/)) / [Python 3.4](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tf_nightly_gpu-1.head-cp34-cp34m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/)) / [Python 3.5](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tf_nightly_gpu-1.head-cp35-cp35m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/)) / [Python 3.6](http://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.6,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tf_nightly_gpu-1.head-cp36-cp36m-linux_x86_64.whl) +* Mac CPU-only: [Python 2](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-mac/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac-slave/lastSuccessfulBuild/artifact/pip_test/whl/tf_nightly-1.head-py2-none-any.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-mac/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac-slave/)) / [Python 3](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-mac/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac-slave/lastSuccessfulBuild/artifact/pip_test/whl/tf_nightly-1.head-py3-none-any.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-mac/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac-slave/)) +* Windows CPU-only: [Python 3.5 64-bit](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows,PY=35/lastSuccessfulBuild/artifact/cmake_build/tf_python/dist/tf_nightly-1.head-cp35-cp35m-win_amd64.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows,PY=35/)) / [Python 3.6 64-bit](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows,PY=36/lastSuccessfulBuild/artifact/cmake_build/tf_python/dist/tf_nightly-1.head-cp36-cp36m-win_amd64.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows,PY=36/)) +* Windows GPU: [Python 3.5 64-bit](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows-gpu,PY=35/lastSuccessfulBuild/artifact/cmake_build/tf_python/dist/tf_nightly_gpu-1.head-cp35-cp35m-win_amd64.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows-gpu,PY=35/)) / [Python 3.6 64-bit](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows-gpu,PY=36/lastSuccessfulBuild/artifact/cmake_build/tf_python/dist/tf_nightly_gpu-1.head-cp36-cp36m-win_amd64.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows-gpu,PY=36/)) +* Android: [demo APK](https://ci.tensorflow.org/view/Nightly/job/nightly-android/lastSuccessfulBuild/artifact/out/tensorflow_demo.apk), [native libs](https://ci.tensorflow.org/view/Nightly/job/nightly-android/lastSuccessfulBuild/artifact/out/native/) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-android/)) #### *Try your first TensorFlow program* @@ -64,11 +73,11 @@ $ python ## For more information -* [TensorFlow website](https://www.tensorflow.org) +* [TensorFlow Website](https://www.tensorflow.org) * [TensorFlow White Papers](https://www.tensorflow.org/about/bib) * [TensorFlow Model Zoo](https://github.com/tensorflow/models) * [TensorFlow MOOC on Udacity](https://www.udacity.com/course/deep-learning--ud730) -* [TensorFlow course at Stanford](https://web.stanford.edu/class/cs20si) +* [TensorFlow Course at Stanford](https://web.stanford.edu/class/cs20si) Learn more about the TensorFlow community at the [community page of tensorflow.org](https://www.tensorflow.org/community) for a few ways to participate. diff --git a/RELEASE.md b/RELEASE.md index ec66e555add9e4ef6da1169754454e4b6ad7acee..e04bd3fc505d51ade9e9fa12c822cb695e90b4f3 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,6 +1,59 @@ # Release 1.4.0 ## Major Features And Improvements +* `tf.keras` is now part of the core TensorFlow API. +* [`tf.data`](http://tensorflow.org/programmers_guide/datasets) is now part of + the core TensorFlow API. + * The API is now subject to backwards compatibility guarantees. + * For a guide to migrating from the `tf.contrib.data` API, see the + [README](https://github.com/tensorflow/tensorflow/blob/r1.4/tensorflow/contrib/data/README.md). + * Major new features include `Dataset.from_generator()` (for building an input + pipeline from a Python generator), and the `Dataset.apply()` method for + applying custom transformation functions. + * Several custom transformation functions have been added, including + `tf.contrib.data.batch_and_drop_remainder()` and + `tf.contrib.data.sloppy_interleave()`. +* Add `train_and_evaluate` for simple distributed `Estimator` training. +* Add `tf.spectral.dct` for computing the DCT-II. +* Add Mel-Frequency Cepstral Coefficient support to `tf.contrib.signal` + (with GPU and gradient support). +* Add a self-check on `import tensorflow` for Windows DLL issues. +* Add NCHW support to `tf.depth_to_space` on GPU. +* TensorFlow Debugger (tfdbg): + * Add `eval` command to allow evaluation of arbitrary Python/numpy expressions + in tfdbg command-line interface. See + [Debugging TensorFlow Programs](https://www.tensorflow.org/programmers_guide/debugger) + for more details. + * Usability improvement: The frequently used tensor filter `has_inf_or_nan` is + now added to `Session` wrappers and hooks by default. So there is no need + for clients to call `.add_tensor_filter(tf_debug.has_inf_or_nan)` anymore. +* SinhArcsinh (scalar) distribution added to `contrib.distributions`. +* Make `GANEstimator` opensource. +* `Estimator.export_savedmodel()` now includes all valid serving signatures + that can be constructed from the Serving Input Receiver and all available + ExportOutputs. For instance, a classifier may provide regression- and + prediction-flavored outputs, in addition to the classification-flavored one. + Building signatures from these allows TF Serving to honor requests using the + different APIs (Classify, Regress, and Predict). Furthermore, + `serving_input_receiver_fn()` may now specify alternative subsets of nodes + that may act as inputs. This allows, for instance, producing a prediction + signature for a classifier that accepts raw `Tensors` instead of a serialized + `tf.Example`. +* Add `tf.contrib.bayesflow.hmc`. +* Add `tf.contrib.distributions.MixtureSameFamily`. +* Make `Dataset.shuffle()` always reshuffles after each iteration by default. +* Add `tf.contrib.bayesflow.metropolis_hastings`. +* Add `log_rate` parameter to `tf.contrib.distributions.Poisson`. +* Extend `tf.contrib.distributions.bijector` API to handle some non-injective + transforms. +* Java: + * Generics (e.g., `Tensor`) for improved type-safety + (courtesy @andrewcmyers). + * Support for multi-dimensional string tensors. + * Support loading of custom operations (e.g. many in `tf.contrib`) on Linux + and OS X +* All our prebuilt binaries have been built with CUDA 8 and cuDNN 6. + We anticipate releasing TensorFlow 1.5 with CUDA 9 and cuDNN 7. ## Bug Fixes and Other Changes * `tf.nn.rnn_cell.DropoutWrapper` is now more careful about dropping out LSTM @@ -9,6 +62,69 @@ for LSTMs and stacked LSTMs. This bug fix follows recommendations from published literature, but is a behavioral change. State dropout behavior may be customized via the new `dropout_state_filter_visitor` argument. +* Removed `tf.contrib.training.python_input`. The same behavior, in a more + flexible and reproducible package, is available via the new + `tf.contrib.data.Dataset.from_generator` method! +* Fix `tf.contrib.distributions.Affine` incorrectly computing log-det-jacobian. +* Fix `tf.random_gamma` incorrectly handling non-batch, scalar draws. +* Resolved a race condition in TensorForest TreePredictionsV4Op. +* Google Cloud Storage file system, Amazon S3 file system, and Hadoop file + system support are now default build options. +* Custom op libraries must link against libtensorflow_framework.so + (installed at `tf.sysconfig.get_lib()`). +* Change `RunConfig` default behavior to not set a random seed, making random + behavior independently random on distributed workers. We expect this to + generally improve training performance. Models that do rely on determinism + should set a random seed explicitly. + +## Breaking Changes to the API +* The signature of the `tf.contrib.data.rejection_resample()` function has been + changed. It now returns a function that can be used as an argument to + `Dataset.apply()`. +* Remove `tf.contrib.data.Iterator.from_dataset()` method. Use + `Dataset.make_initializable_iterator()` instead. +* Remove seldom used and unnecessary `tf.contrib.data.Iterator.dispose_op()`. +* Reorder some TFGAN loss functions in a non-backwards compatible way. + +## Known Issues +* In Python 3, `Dataset.from_generator()` does not support Unicode strings. + You must convert any strings to bytes objects before yielding them from + the generator. + +## Thanks to our Contributors + +This release contains contributions from many people at Google, as well as: + +4d55397500, Abdullah Alrasheed, abenmao, Adam Salvail, Aditya Dhulipala, Ag Ramesh, +Akimasa Kimura, Alan Du, Alan Yee, Alexander, Amit Kushwaha, Amy, Andrei Costinescu, +Andrei Nigmatulin, Andrew Erlichson, Andrew Myers, Andrew Stepanov, Androbin, AngryPowman, +Anish Shah, Anton Daitche, Artsiom Chapialiou, asdf2014, Aseem Raj Baranwal, Ash Hall, +Bart Kiers, Batchu Venkat Vishal, ben, Ben Barsdell, Bill Piel, Carl Thomé, Catalin Voss, +Changming Sun, Chengzhi Chen, Chi Zeng, Chris Antaki, Chris Donahue, Chris Oelmueller, +Chris Tava, Clayne Robison, Codrut, Courtial Florian, Dalmo Cirne, Dan J, Darren Garvey, +David Kristoffersson, David Norman, David RöThlisberger, DavidNorman, Dhruv, DimanNe, +Dorokhov, Duncan Mac-Vicar P, EdwardDixon, EMCP, error.d, FAIJUL, Fan Xia, +Francois Xavier, Fred Reiss, Freedom" Koan-Sin Tan, Fritz Obermeyer, Gao, Xiang, +Guenther Schmuelling, Guo Yejun (郭叶军), Hans Gaiser, HectorSVC, Hyungsuk Yoon, +James Pruegsanusak, Jay Young, Jean Wanka, Jeff Carpenter, Jeremy Rutman, Jeroen BéDorf, +Jett Jones, Jimmy Jia, jinghuangintel, jinze1994, JKurland, Joel Hestness, joetoth, +John B Nelson, John Impallomeni, John Lawson, Jonas, Jonathan Dekhtiar, joshkyh, Jun Luan, +Jun Mei, Kai Sasaki, Karl Lessard, karl@kubx.ca, Kb Sriram, Kenichi Ueno, Kevin Slagle, +Kongsea, Lakshay Garg, lhlmgr, Lin Min, liu.guangcong, Loki Der Quaeler, Louie Helm, +lucasmoura, Luke Iwanski, Lyndon White, Mahmoud Abuzaina, Marcel Puyat, Mark Aaron Shirley, +Michele Colombo, MtDersvan, Namrata-Ibm, Nathan Luehr, Naurril, Nayana Thorat, Nicolas Lopez, +Niranjan Hasabnis, Nolan Liu, Nouce, Oliver Hennigh, osdamv, Patrik Erdes, +Patryk Chrabaszcz, Pavel Christof, Penghao Cen, postBG, Qingqing Cao, Qingying Chen, qjivy, +Raphael, Rasmi, raymondxyang, Renze Yu, resec, Roffel, Ruben Vereecken, Ryohei Kuroki, +sandipmgiri, Santiago Castro, Scott Kirkland, Sean Vig, Sebastian Raschka, Sebastian Weiss, +Sergey Kolesnikov, Sergii Khomenko, Shahid, Shivam Kotwalia, Stuart Berg, Sumit Gouthaman, +superzerg, Sven Mayer, tetris, Ti Zhou, Tiago Freitas Pereira, Tian Jin, Tomoaki Oiki, +Vaibhav Sood, vfdev, Vivek Rane, Vladimir Moskva, wangqr, Weber Xie, Will Frey, +Yan Facai (颜发才), yanivbl6, Yaroslav Bulatov, Yixing Lao, Yong Tang, youkaichao, +Yuan (Terry) Tang, Yue Zhang, Yuxin Wu, Ziming Dong, ZxYuan, 黄璞 + +We are also grateful to all who filed issues or helped resolve them, asked and +answered questions, and were part of inspiring discussions. # Release 1.3.0 @@ -43,7 +159,7 @@ See also [TensorBoard 0.1.4](https://github.com/tensorflow/tensorboard/releases/ * Display feed values with the `print_feed` or `pf` command and clickable links in the curses UI. * Runtime profiler at the op level and the Python source line level with the `run -p` command. * Initial release of the statistical distribution library `tf.distributions`. -* GPU kernels and speed improvements for for unary `tf.where` and `tf.nn.top_k`. +* GPU kernels and speed improvements for unary `tf.where` and `tf.nn.top_k`. * Monotonic Attention wrappers added to `tf.contrib.seq2seq`. * Added `tf.contrib.signal`, a library for signal processing primitives. * Added `tf.contrib.resampler`, containing CPU and GPU ops for differentiable resampling of images. @@ -378,7 +494,7 @@ answered questions, and were part of inspiring discussions. This release contains contributions from many people at Google, as well as: A. Besir Kurtulmus, Adal Chiriliuc, @akash, Alec-Desouza, Alex Rothberg, Alex -Sergeev, Alexander Heinecke, Allen Guo, Andreas Madsen, Ankesh Anand, Anton +Sergeev, Alexander Heinecke, Allen Guo, Andreas Madsen, Ankesh Anand, Anton Loss, @Aravind, @Arie, Ashutosh Das, AuréLien Geron, Bairen Yi, @bakunyo, Ben Visser, Brady Zhou, Calpa Liu, Changming Sun, Chih Cheng Liang, Christopher Berner, Clark Zinzow, @Conchylicultor, Dan Ellis, Dan J, Dan Jarvis, Daniel diff --git a/WORKSPACE b/WORKSPACE index 5e9b991fccaa8d11e9233c6c2db08d4798168796..b40913801ba8e3c8ee73f7ba69540b520ad698a6 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -2,11 +2,11 @@ workspace(name = "org_tensorflow") http_archive( name = "io_bazel_rules_closure", - sha256 = "bc41b80486413aaa551860fc37471dbc0666e1dbb5236fb6177cb83b0c105846", - strip_prefix = "rules_closure-dec425a4ff3faf09a56c85d082e4eed05d8ce38f", + sha256 = "110fe68753413777944b473c25eed6368c4a0487cee23a7bac1b13cc49d3e257", + strip_prefix = "rules_closure-4af89ef1db659eb41f110df189b67d4cf14073e1", urls = [ - "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz", # 2017-06-02 - "https://github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/4af89ef1db659eb41f110df189b67d4cf14073e1.tar.gz", + "https://github.com/bazelbuild/rules_closure/archive/4af89ef1db659eb41f110df189b67d4cf14073e1.tar.gz", # 2017-08-28 ], ) @@ -22,12 +22,11 @@ load("//tensorflow:workspace.bzl", "tf_workspace") # api_level = 23, # # Ensure that you have the build_tools_version below installed in the # # SDK manager as it updates periodically. -# build_tools_version = "25.0.2", +# build_tools_version = "26.0.1", # # Replace with path to Android SDK on your system # path = "", #) # -# Android NDK r12b is recommended (higher may cause issues with Bazel) #android_ndk_repository( # name="androidndk", # path="", diff --git a/configure.py b/configure.py index 1a0f71ed94358ee8ea81a13b192d4131d6133f66..e4218b56518a3d76564533df9b31d514a7313c49 100644 --- a/configure.py +++ b/configure.py @@ -25,14 +25,19 @@ import re import subprocess import sys +# pylint: disable=g-import-not-at-top try: from shutil import which except ImportError: from distutils.spawn import find_executable as which - -_TF_BAZELRC = '.tf_configure.bazelrc' -_DEFAULT_CUDA_VERSION = '8.0' -_DEFAULT_CUDNN_VERSION = '6' +# pylint: enable=g-import-not-at-top + +_TF_BAZELRC = os.path.join(os.path.dirname(os.path.abspath(__file__)), + '.tf_configure.bazelrc') +_TF_WORKSPACE = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'WORKSPACE') +_DEFAULT_CUDA_VERSION = '9.0' +_DEFAULT_CUDNN_VERSION = '7' _DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,5.2' _DEFAULT_CUDA_PATH = '/usr/local/cuda' _DEFAULT_CUDA_PATH_LINUX = '/opt/cuda' @@ -40,6 +45,14 @@ _DEFAULT_CUDA_PATH_WIN = ('C:/Program Files/NVIDIA GPU Computing ' 'Toolkit/CUDA/v%s' % _DEFAULT_CUDA_VERSION) _TF_OPENCL_VERSION = '1.2' _DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp' +_DEFAULT_TRISYCL_INCLUDE_DIR = '/usr/local/triSYCL/include' +_SUPPORTED_ANDROID_NDK_VERSIONS = [10, 11, 12, 13, 14, 15] + +_DEFAULT_PROMPT_ASK_ATTEMPTS = 10 + + +class UserInputError(Exception): + pass def is_windows(): @@ -143,7 +156,7 @@ def run_shell(cmd, allow_non_zero=False): def cygpath(path): """Convert path from posix to windows.""" - return run_shell(['cygpath', '-m', path]) + return os.path.abspath(path).replace('\\', '/') def get_python_path(environ_cp, python_bin_path): @@ -154,7 +167,7 @@ def get_python_path(environ_cp, python_bin_path): try: library_paths = run_shell( [python_bin_path, '-c', - 'import site; print("\\n".join(site.getsitepackages()))']).split("\n") + 'import site; print("\\n".join(site.getsitepackages()))']).split('\n') except subprocess.CalledProcessError: library_paths = [run_shell( [python_bin_path, '-c', @@ -175,7 +188,7 @@ def get_python_major_version(python_bin_path): return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])']) -def setup_python(environ_cp, bazel_version): +def setup_python(environ_cp): """Setup python related env variables.""" # Get PYTHON_BIN_PATH, default is the current running python. default_python_bin_path = sys.executable @@ -196,7 +209,7 @@ def setup_python(environ_cp, bazel_version): environ_cp['PYTHON_BIN_PATH'] = '' # Convert python path to Windows style before checking lib and version - if is_cygwin(): + if is_windows() or is_cygwin(): python_bin_path = cygpath(python_bin_path) # Get PYTHON_LIB_PATH @@ -219,33 +232,15 @@ def setup_python(environ_cp, bazel_version): python_major_version = get_python_major_version(python_bin_path) # Convert python path to Windows style before writing into bazel.rc - if is_cygwin(): + if is_windows() or is_cygwin(): python_lib_path = cygpath(python_lib_path) # Set-up env variables used by python_configure.bzl write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path) write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path) - write_to_bazelrc('build --define PYTHON_BIN_PATH="%s"' % python_bin_path) - write_to_bazelrc('build --define PYTHON_LIB_PATH="%s"' % python_lib_path) write_to_bazelrc('build --force_python=py%s' % python_major_version) write_to_bazelrc('build --host_force_python=py%s' % python_major_version) - bazel_version_int = convert_version_to_int(bazel_version) - version_0_5_3_int = convert_version_to_int('0.5.3') - # If bazel_version_int is None, we are testing a release Bazel, then the - # version should be higher than 0.5.3 - # TODO(pcloudy): remove this after required min bazel version is higher - # than 0.5.3 - if not bazel_version_int or bazel_version_int >= version_0_5_3_int: - write_to_bazelrc('build --python_path=\"%s"' % python_bin_path) - else: - write_to_bazelrc('build --python%s_path=\"%s"' % (python_major_version, - python_bin_path)) - write_to_bazelrc('test --force_python=py%s' % python_major_version) - write_to_bazelrc('test --host_force_python=py%s' % python_major_version) - write_to_bazelrc('test --define PYTHON_BIN_PATH="%s"' % python_bin_path) - write_to_bazelrc('test --define PYTHON_LIB_PATH="%s"' % python_lib_path) - write_to_bazelrc('run --define PYTHON_BIN_PATH="%s"' % python_bin_path) - write_to_bazelrc('run --define PYTHON_LIB_PATH="%s"' % python_lib_path) + write_to_bazelrc('build --python_path=\"%s"' % python_bin_path) environ_cp['PYTHON_BIN_PATH'] = python_bin_path # Write tools/python_bin_path.sh @@ -261,7 +256,7 @@ def reset_tf_configure_bazelrc(): if not os.path.exists('.bazelrc'): if os.path.exists(os.path.join(home, '.bazelrc')): with open('.bazelrc', 'a') as f: - f.write('import %s/.bazelrc\n' % home) + f.write('import %s/.bazelrc\n' % home.replace('\\', '/')) else: open('.bazelrc', 'w').close() @@ -270,19 +265,6 @@ def reset_tf_configure_bazelrc(): f.write('import %workspace%/.tf_configure.bazelrc\n') -def run_gen_git_source(environ_cp): - """Run the gen_git_source to create links. - - The links are for bazel to track dependencies for git hash propagation. - - Args: - environ_cp: copy of the os.environ. - """ - cmd = '"%s" tensorflow/tools/git/gen_git_source.py --configure %s' % ( - environ_cp.get('PYTHON_BIN_PATH'), os.getcwd()) - os.system(cmd) - - def cleanup_makefile(): """Delete any leftover BUILD files from the Makefile build. @@ -359,7 +341,7 @@ def get_var(environ_cp, def set_build_var(environ_cp, var_name, query_item, option_name, - enabled_by_default): + enabled_by_default, bazel_config_name=None): """Set if query_item will be enabled for the build. Ask user if query_item will be enabled. Default is used if no input is given. @@ -372,12 +354,18 @@ def set_build_var(environ_cp, var_name, query_item, option_name, System". option_name: string for option to define in .bazelrc. enabled_by_default: boolean for default behavior. + bazel_config_name: Name for Bazel --config argument to enable build feature. """ var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default))) environ_cp[var_name] = var if var == '1': write_to_bazelrc('build --define %s=true' % option_name) + elif bazel_config_name is not None: + # TODO(mikecase): Migrate all users of configure.py to use --config Bazel + # options and not to set build configs through environment variables. + write_to_bazelrc('build:%s --define %s=true' + % (bazel_config_name, option_name)) def set_action_env_var(environ_cp, @@ -488,7 +476,14 @@ def set_cc_opt_flags(environ_cp): cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS', question, default_cc_opt_flags) for opt in cc_opt_flags.split(): - write_to_bazelrc('build:opt --cxxopt=%s --copt=%s' % (opt, opt)) + write_to_bazelrc('build:opt --copt=%s' % opt) + # It should be safe on the same build host. + write_to_bazelrc('build:opt --host_copt=-march=native') + write_to_bazelrc('build:opt --define with_default_optimizations=true') + # TODO(mikecase): Remove these default defines once we are able to get + # TF Lite targets building without them. + write_to_bazelrc('build --copt=-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK') + write_to_bazelrc('build --host_copt=-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK') def set_tf_cuda_clang(environ_cp): @@ -510,6 +505,21 @@ def set_tf_cuda_clang(environ_cp): no_reply=no_reply) +def set_tf_download_clang(environ_cp): + """Set TF_DOWNLOAD_CLANG action_env.""" + question = 'Do you want to download a fresh release of clang? (Experimental)' + yes_reply = 'Clang will be downloaded and used to compile tensorflow.' + no_reply = 'Clang will not be downloaded.' + set_action_env_var( + environ_cp, + 'TF_DOWNLOAD_CLANG', + None, + False, + question=question, + yes_reply=yes_reply, + no_reply=no_reply) + + def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var, var_default): """Get var_name either from env, or user or default. @@ -558,6 +568,218 @@ def set_clang_cuda_compiler_path(environ_cp): clang_cuda_compiler_path) +def prompt_loop_or_load_from_env( + environ_cp, + var_name, + var_default, + ask_for_var, + check_success, + error_msg, + suppress_default_error=False, + n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS +): + """Loop over user prompts for an ENV param until receiving a valid response. + + For the env param var_name, read from the environment or verify user input + until receiving valid input. When done, set var_name in the environ_cp to its + new value. + + Args: + environ_cp: (Dict) copy of the os.environ. + var_name: (String) string for name of environment variable, e.g. "TF_MYVAR". + var_default: (String) default value string. + ask_for_var: (String) string for how to ask for user input. + check_success: (Function) function that takes one argument and returns a + boolean. Should return True if the value provided is considered valid. May + contain a complex error message if error_msg does not provide enough + information. In that case, set suppress_default_error to True. + error_msg: (String) String with one and only one '%s'. Formatted with each + invalid response upon check_success(input) failure. + suppress_default_error: (Bool) Suppress the above error message in favor of + one from the check_success function. + n_ask_attempts: (Integer) Number of times to query for valid input before + raising an error and quitting. + + Returns: + [String] The value of var_name after querying for input. + + Raises: + UserInputError: if a query has been attempted n_ask_attempts times without + success, assume that the user has made a scripting error, and will continue + to provide invalid input. Raise the error to avoid infinitely looping. + """ + default = environ_cp.get(var_name) or var_default + full_query = '%s [Default is %s]: ' % ( + ask_for_var, + default, + ) + + for _ in range(n_ask_attempts): + val = get_from_env_or_user_or_default(environ_cp, + var_name, + full_query, + default) + if check_success(val): + break + if not suppress_default_error: + print(error_msg % val) + environ_cp[var_name] = '' + else: + raise UserInputError('Invalid %s setting was provided %d times in a row. ' + 'Assuming to be a scripting mistake.' % + (var_name, n_ask_attempts)) + + environ_cp[var_name] = val + return val + + +def create_android_ndk_rule(environ_cp): + """Set ANDROID_NDK_HOME and write Android NDK WORKSPACE rule.""" + if is_windows() or is_cygwin(): + default_ndk_path = cygpath('%s/Android/Sdk/ndk-bundle' % + environ_cp['APPDATA']) + elif is_macos(): + default_ndk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME'] + else: + default_ndk_path = '%s/Android/Sdk/ndk-bundle' % environ_cp['HOME'] + + def valid_ndk_path(path): + return (os.path.exists(path) and + os.path.exists(os.path.join(path, 'source.properties'))) + + android_ndk_home_path = prompt_loop_or_load_from_env( + environ_cp, + var_name='ANDROID_NDK_HOME', + var_default=default_ndk_path, + ask_for_var='Please specify the home path of the Android NDK to use.', + check_success=valid_ndk_path, + error_msg=('The path %s or its child file "source.properties" ' + 'does not exist.') + ) + + write_android_ndk_workspace_rule(android_ndk_home_path) + + +def create_android_sdk_rule(environ_cp): + """Set Android variables and write Android SDK WORKSPACE rule.""" + if is_windows() or is_cygwin(): + default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA']) + elif is_macos(): + default_sdk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME'] + else: + default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME'] + + def valid_sdk_path(path): + return (os.path.exists(path) and + os.path.exists(os.path.join(path, 'platforms')) and + os.path.exists(os.path.join(path, 'build-tools'))) + + android_sdk_home_path = prompt_loop_or_load_from_env( + environ_cp, + var_name='ANDROID_SDK_HOME', + var_default=default_sdk_path, + ask_for_var='Please specify the home path of the Android SDK to use.', + check_success=valid_sdk_path, + error_msg=('Either %s does not exist, or it does not contain the ' + 'subdirectories "platforms" and "build-tools".')) + + platforms = os.path.join(android_sdk_home_path, 'platforms') + api_levels = sorted(os.listdir(platforms)) + api_levels = [x.replace('android-', '') for x in api_levels] + + def valid_api_level(api_level): + return os.path.exists(os.path.join(android_sdk_home_path, + 'platforms', + 'android-' + api_level)) + + android_api_level = prompt_loop_or_load_from_env( + environ_cp, + var_name='ANDROID_API_LEVEL', + var_default=api_levels[-1], + ask_for_var=('Please specify the Android SDK API level to use. ' + '[Available levels: %s]') % api_levels, + check_success=valid_api_level, + error_msg='Android-%s is not present in the SDK path.') + + build_tools = os.path.join(android_sdk_home_path, 'build-tools') + versions = sorted(os.listdir(build_tools)) + + def valid_build_tools(version): + return os.path.exists(os.path.join(android_sdk_home_path, + 'build-tools', + version)) + + android_build_tools_version = prompt_loop_or_load_from_env( + environ_cp, + var_name='ANDROID_BUILD_TOOLS_VERSION', + var_default=versions[-1], + ask_for_var=('Please specify an Android build tools version to use. ' + '[Available versions: %s]') % versions, + check_success=valid_build_tools, + error_msg=('The selected SDK does not have build-tools version %s ' + 'available.')) + + write_android_sdk_workspace_rule(android_sdk_home_path, + android_build_tools_version, + android_api_level) + + +def write_android_sdk_workspace_rule(android_sdk_home_path, + android_build_tools_version, + android_api_level): + print('Writing android_sdk_workspace rule.\n') + with open(_TF_WORKSPACE, 'a') as f: + f.write(""" +android_sdk_repository( + name="androidsdk", + api_level=%s, + path="%s", + build_tools_version="%s")\n +""" % (android_api_level, android_sdk_home_path, android_build_tools_version)) + + +def write_android_ndk_workspace_rule(android_ndk_home_path): + print('Writing android_ndk_workspace rule.') + ndk_api_level = check_ndk_level(android_ndk_home_path) + if int(ndk_api_level) not in _SUPPORTED_ANDROID_NDK_VERSIONS: + print('WARNING: The API level of the NDK in %s is %s, which is not ' + 'supported by Bazel (officially supported versions: %s). Please use ' + 'another version. Compiling Android targets may result in confusing ' + 'errors.\n' % (android_ndk_home_path, ndk_api_level, + _SUPPORTED_ANDROID_NDK_VERSIONS)) + with open(_TF_WORKSPACE, 'a') as f: + f.write(""" +android_ndk_repository( + name="androidndk", + path="%s", + api_level=%s)\n +""" % (android_ndk_home_path, ndk_api_level)) + + +def check_ndk_level(android_ndk_home_path): + """Check the revision number of an Android NDK path.""" + properties_path = '%s/source.properties' % android_ndk_home_path + if is_windows() or is_cygwin(): + properties_path = cygpath(properties_path) + with open(properties_path, 'r') as f: + filedata = f.read() + + revision = re.search(r'Pkg.Revision = (\d+)', filedata) + if revision: + return revision.group(1) + return None + + +def workspace_has_any_android_rule(): + """Check the WORKSPACE for existing android_*_repository rules.""" + with open(_TF_WORKSPACE, 'r') as f: + workspace = f.read() + has_any_rule = re.search(r'^android_[ns]dk_repository', + workspace, + re.MULTILINE) + return has_any_rule + + def set_gcc_host_compiler_path(environ_cp): """Set GCC_HOST_COMPILER_PATH.""" default_gcc_host_compiler_path = which('gcc') or '' @@ -567,23 +789,16 @@ def set_gcc_host_compiler_path(environ_cp): # os.readlink is only available in linux default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink) - ask_gcc_path = ( - 'Please specify which gcc should be used by nvcc as the ' - 'host compiler. [Default is %s]: ') % default_gcc_host_compiler_path - while True: - gcc_host_compiler_path = get_from_env_or_user_or_default( - environ_cp, 'GCC_HOST_COMPILER_PATH', ask_gcc_path, - default_gcc_host_compiler_path) - - if os.path.exists(gcc_host_compiler_path): - break - - # Reset and retry - print('Invalid gcc path. %s cannot be found' % gcc_host_compiler_path) - environ_cp['GCC_HOST_COMPILER_PATH'] = '' + gcc_host_compiler_path = prompt_loop_or_load_from_env( + environ_cp, + var_name='GCC_HOST_COMPILER_PATH', + var_default=default_gcc_host_compiler_path, + ask_for_var= + 'Please specify which gcc should be used by nvcc as the host compiler.', + check_success=os.path.exists, + error_msg='Invalid gcc path. %s cannot be found.', + ) - # Set GCC_HOST_COMPILER_PATH - environ_cp['GCC_HOST_COMPILER_PATH'] = gcc_host_compiler_path write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path) @@ -593,14 +808,14 @@ def set_tf_cuda_version(environ_cp): 'Please specify the CUDA SDK version you want to use, ' 'e.g. 7.0. [Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION - while True: + for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS): # Configure the Cuda SDK version to use. tf_cuda_version = get_from_env_or_user_or_default( environ_cp, 'TF_CUDA_VERSION', ask_cuda_version, _DEFAULT_CUDA_VERSION) # Find out where the CUDA toolkit is installed default_cuda_path = _DEFAULT_CUDA_PATH - if is_cygwin(): + if is_windows() or is_cygwin(): default_cuda_path = cygpath( environ_cp.get('CUDA_PATH', _DEFAULT_CUDA_PATH_WIN)) elif is_linux(): @@ -631,6 +846,11 @@ def set_tf_cuda_version(environ_cp): environ_cp['TF_CUDA_VERSION'] = '' environ_cp['CUDA_TOOLKIT_PATH'] = '' + else: + raise UserInputError('Invalid TF_CUDA_SETTING setting was provided %d ' + 'times in a row. Assuming to be a scripting mistake.' % + _DEFAULT_PROMPT_ASK_ATTEMPTS) + # Set CUDA_TOOLKIT_PATH and TF_CUDA_VERSION environ_cp['CUDA_TOOLKIT_PATH'] = cuda_toolkit_path write_action_env_to_bazelrc('CUDA_TOOLKIT_PATH', cuda_toolkit_path) @@ -638,13 +858,13 @@ def set_tf_cuda_version(environ_cp): write_action_env_to_bazelrc('TF_CUDA_VERSION', tf_cuda_version) -def set_tf_cunn_version(environ_cp): +def set_tf_cudnn_version(environ_cp): """Set CUDNN_INSTALL_PATH and TF_CUDNN_VERSION.""" ask_cudnn_version = ( 'Please specify the cuDNN version you want to use. ' '[Leave empty to default to cuDNN %s.0]: ') % _DEFAULT_CUDNN_VERSION - while True: + for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS): tf_cudnn_version = get_from_env_or_user_or_default( environ_cp, 'TF_CUDNN_VERSION', ask_cudnn_version, _DEFAULT_CUDNN_VERSION) @@ -660,7 +880,7 @@ def set_tf_cunn_version(environ_cp): # unusable. Going through one more level of expansion to handle that. cudnn_install_path = os.path.realpath( os.path.expanduser(cudnn_install_path)) - if is_cygwin(): + if is_windows() or is_cygwin(): cudnn_install_path = cygpath(cudnn_install_path) if is_windows(): @@ -685,10 +905,13 @@ def set_tf_cunn_version(environ_cp): ldconfig_bin = which('ldconfig') or '/sbin/ldconfig' cudnn_path_from_ldconfig = run_shell([ldconfig_bin, '-p']) cudnn_path_from_ldconfig = re.search('.*libcudnn.so .* => (.*)', - cudnn_path_from_ldconfig).group(1) - if os.path.exists('%s.%s' % (cudnn_path_from_ldconfig, tf_cudnn_version)): - cudnn_install_path = os.path.dirname(cudnn_path_from_ldconfig) - break + cudnn_path_from_ldconfig) + if cudnn_path_from_ldconfig: + cudnn_path_from_ldconfig = cudnn_path_from_ldconfig.group(1) + if os.path.exists('%s.%s' % (cudnn_path_from_ldconfig, + tf_cudnn_version)): + cudnn_install_path = os.path.dirname(cudnn_path_from_ldconfig) + break # Reset and Retry print( @@ -700,6 +923,10 @@ def set_tf_cunn_version(environ_cp): print('%s.%s' % (cudnn_path_from_ldconfig, tf_cudnn_version)) environ_cp['TF_CUDNN_VERSION'] = '' + else: + raise UserInputError('Invalid TF_CUDNN setting was provided %d ' + 'times in a row. Assuming to be a scripting mistake.' % + _DEFAULT_PROMPT_ASK_ATTEMPTS) # Set CUDNN_INSTALL_PATH and TF_CUDNN_VERSION environ_cp['CUDNN_INSTALL_PATH'] = cudnn_install_path @@ -808,102 +1035,115 @@ def set_other_cuda_vars(environ_cp): def set_host_cxx_compiler(environ_cp): """Set HOST_CXX_COMPILER.""" default_cxx_host_compiler = which('g++') or '' - ask_cxx_host_compiler = ( - 'Please specify which C++ compiler should be used as' - ' the host C++ compiler. [Default is %s]: ') % default_cxx_host_compiler - while True: - host_cxx_compiler = get_from_env_or_user_or_default( - environ_cp, 'HOST_CXX_COMPILER', ask_cxx_host_compiler, - default_cxx_host_compiler) - if os.path.exists(host_cxx_compiler): - break - - # Reset and retry - print('Invalid C++ compiler path. %s cannot be found' % host_cxx_compiler) - environ_cp['HOST_CXX_COMPILER'] = '' + host_cxx_compiler = prompt_loop_or_load_from_env( + environ_cp, + var_name='HOST_CXX_COMPILER', + var_default=default_cxx_host_compiler, + ask_for_var=('Please specify which C++ compiler should be used as the ' + 'host C++ compiler.'), + check_success=os.path.exists, + error_msg='Invalid C++ compiler path. %s cannot be found.', + ) - # Set HOST_CXX_COMPILER - environ_cp['HOST_CXX_COMPILER'] = host_cxx_compiler write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler) def set_host_c_compiler(environ_cp): """Set HOST_C_COMPILER.""" default_c_host_compiler = which('gcc') or '' - ask_c_host_compiler = ( - 'Please specify which C compiler should be used as the' - ' host C compiler. [Default is %s]: ') % default_c_host_compiler - - while True: - host_c_compiler = get_from_env_or_user_or_default( - environ_cp, 'HOST_C_COMPILER', ask_c_host_compiler, - default_c_host_compiler) - if os.path.exists(host_c_compiler): - break - # Reset and retry - print('Invalid C compiler path. %s cannot be found' % host_c_compiler) - environ_cp['HOST_C_COMPILER'] = '' + host_c_compiler = prompt_loop_or_load_from_env( + environ_cp, + var_name='HOST_C_COMPILER', + var_default=default_c_host_compiler, + ask_for_var=('Please specify which C compiler should be used as the host' + 'C compiler.'), + check_success=os.path.exists, + error_msg='Invalid C compiler path. %s cannot be found.', + ) - # Set HOST_C_COMPILER - environ_cp['HOST_C_COMPILER'] = host_c_compiler write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler) def set_computecpp_toolkit_path(environ_cp): """Set COMPUTECPP_TOOLKIT_PATH.""" - ask_computecpp_toolkit_path = ('Please specify the location where ComputeCpp ' - 'for SYCL %s is installed. [Default is %s]: ' - ) % (_TF_OPENCL_VERSION, - _DEFAULT_COMPUTECPP_TOOLKIT_PATH) - while True: - computecpp_toolkit_path = get_from_env_or_user_or_default( - environ_cp, 'COMPUTECPP_TOOLKIT_PATH', ask_computecpp_toolkit_path, - _DEFAULT_COMPUTECPP_TOOLKIT_PATH) + def toolkit_exists(toolkit_path): + """Check if a computecpp toolkit path is valid.""" if is_linux(): sycl_rt_lib_path = 'lib/libComputeCpp.so' else: sycl_rt_lib_path = '' - sycl_rt_lib_path_full = os.path.join(computecpp_toolkit_path, + sycl_rt_lib_path_full = os.path.join(toolkit_path, sycl_rt_lib_path) - if os.path.exists(sycl_rt_lib_path_full): - break + exists = os.path.exists(sycl_rt_lib_path_full) + if not exists: + print('Invalid SYCL %s library path. %s cannot be found' % + (_TF_OPENCL_VERSION, sycl_rt_lib_path_full)) + return exists - print('Invalid SYCL %s library path. %s cannot be found' % - (_TF_OPENCL_VERSION, sycl_rt_lib_path_full)) - environ_cp['COMPUTECPP_TOOLKIT_PATH'] = '' + computecpp_toolkit_path = prompt_loop_or_load_from_env( + environ_cp, + var_name='COMPUTECPP_TOOLKIT_PATH', + var_default=_DEFAULT_COMPUTECPP_TOOLKIT_PATH, + ask_for_var=( + 'Please specify the location where ComputeCpp for SYCL %s is ' + 'installed.' % _TF_OPENCL_VERSION), + check_success=toolkit_exists, + error_msg='Invalid SYCL compiler path. %s cannot be found.', + suppress_default_error=True) - # Set COMPUTECPP_TOOLKIT_PATH - environ_cp['COMPUTECPP_TOOLKIT_PATH'] = computecpp_toolkit_path write_action_env_to_bazelrc('COMPUTECPP_TOOLKIT_PATH', computecpp_toolkit_path) +def set_trisycl_include_dir(environ_cp): + """Set TRISYCL_INCLUDE_DIR""" + ask_trisycl_include_dir = ('Please specify the location of the triSYCL ' + 'include directory. (Use --config=sycl_trisycl ' + 'when building with Bazel) ' + '[Default is %s]: ' + ) % (_DEFAULT_TRISYCL_INCLUDE_DIR) + while True: + trisycl_include_dir = get_from_env_or_user_or_default( + environ_cp, 'TRISYCL_INCLUDE_DIR', ask_trisycl_include_dir, + _DEFAULT_TRISYCL_INCLUDE_DIR) + if os.path.exists(trisycl_include_dir): + break + + print('Invalid triSYCL include directory, %s cannot be found' + % (trisycl_include_dir)) + + # Set TRISYCL_INCLUDE_DIR + environ_cp['TRISYCL_INCLUDE_DIR'] = trisycl_include_dir + write_action_env_to_bazelrc('TRISYCL_INCLUDE_DIR', + trisycl_include_dir) + def set_mpi_home(environ_cp): """Set MPI_HOME.""" + default_mpi_home = which('mpirun') or which('mpiexec') or '' default_mpi_home = os.path.dirname(os.path.dirname(default_mpi_home)) - ask_mpi_home = ('Please specify the MPI toolkit folder. [Default is %s]: ' - ) % default_mpi_home - while True: - mpi_home = get_from_env_or_user_or_default(environ_cp, 'MPI_HOME', - ask_mpi_home, default_mpi_home) - - if os.path.exists(os.path.join(mpi_home, 'include')) and os.path.exists( - os.path.join(mpi_home, 'lib')): - break - - print('Invalid path to the MPI Toolkit. %s or %s cannot be found' % - (os.path.join(mpi_home, 'include'), - os.path.exists(os.path.join(mpi_home, 'lib')))) - environ_cp['MPI_HOME'] = '' + def valid_mpi_path(mpi_home): + exists = (os.path.exists(os.path.join(mpi_home, 'include')) and + os.path.exists(os.path.join(mpi_home, 'lib'))) + if not exists: + print('Invalid path to the MPI Toolkit. %s or %s cannot be found' % + (os.path.join(mpi_home, 'include'), + os.path.exists(os.path.join(mpi_home, 'lib')))) + return exists - # Set MPI_HOME - environ_cp['MPI_HOME'] = str(mpi_home) + _ = prompt_loop_or_load_from_env( + environ_cp, + var_name='MPI_HOME', + var_default=default_mpi_home, + ask_for_var='Please specify the MPI toolkit folder.', + check_success=valid_mpi_path, + error_msg='', + suppress_default_error=True) def set_other_mpi_vars(environ_cp): @@ -941,13 +1181,51 @@ def set_other_mpi_vars(environ_cp): def set_mkl(): write_to_bazelrc('build:mkl --define using_mkl=true') write_to_bazelrc('build:mkl -c opt') - write_to_bazelrc('build:mkl --copt="-DEIGEN_USE_VML"') print( 'Add "--config=mkl" to your bazel command to build with MKL ' 'support.\nPlease note that MKL on MacOS or windows is still not ' 'supported.\nIf you would like to use a local MKL instead of ' 'downloading, please set the environment variable \"TF_MKL_ROOT\" every ' - 'time before build.') + 'time before build.\n') + + +def set_monolithic(): + # Add --config=monolithic to your bazel command to use a mostly-static + # build and disable modular op registration support (this will revert to + # loading TensorFlow with RTLD_GLOBAL in Python). By default (without + # --config=monolithic), TensorFlow will build with a dependence on + # //tensorflow:libtensorflow_framework.so. + write_to_bazelrc('build:monolithic --define framework_shared_object=false') + # For projects which use TensorFlow as part of a Bazel build process, putting + # nothing in a bazelrc will default to a monolithic build. The following line + # opts in to modular op registration support by default: + write_to_bazelrc('build --define framework_shared_object=true') + + +def create_android_bazelrc_configs(): + # Flags for --config=android + write_to_bazelrc('build:android --crosstool_top=//external:android/crosstool') + write_to_bazelrc( + 'build:android --host_crosstool_top=@bazel_tools//tools/cpp:toolchain') + # Flags for --config=android_arm + write_to_bazelrc('build:android_arm --config=android') + write_to_bazelrc('build:android_arm --cpu=armeabi-v7a') + # Flags for --config=android_arm64 + write_to_bazelrc('build:android_arm64 --config=android') + write_to_bazelrc('build:android_arm64 --cpu=arm64-v8a') + + +def set_grpc_build_flags(): + write_to_bazelrc('build --define grpc_no_ares=true') + +def set_windows_build_flags(): + if is_windows(): + # The non-monolithic build is not supported yet + write_to_bazelrc('build --config monolithic') + # Suppress warning messages + write_to_bazelrc('build --copt=-w --host_copt=-w') + # Output more verbose information when something goes wrong + write_to_bazelrc('build --verbose_failures') def main(): @@ -955,17 +1233,19 @@ def main(): # environment variables. environ_cp = dict(os.environ) - bazel_version = check_bazel_version('0.4.5') + check_bazel_version('0.5.4') reset_tf_configure_bazelrc() cleanup_makefile() - setup_python(environ_cp, bazel_version) - run_gen_git_source(environ_cp) + setup_python(environ_cp) if is_windows(): + environ_cp['TF_NEED_S3'] = '0' environ_cp['TF_NEED_GCP'] = '0' environ_cp['TF_NEED_HDFS'] = '0' environ_cp['TF_NEED_JEMALLOC'] = '0' + environ_cp['TF_NEED_OPENCL_SYCL'] = '0' + environ_cp['TF_NEED_COMPUTECPP'] = '0' environ_cp['TF_NEED_OPENCL'] = '0' environ_cp['TF_CUDA_CLANG'] = '0' @@ -975,32 +1255,50 @@ def main(): set_build_var(environ_cp, 'TF_NEED_JEMALLOC', 'jemalloc as malloc', 'with_jemalloc', True) set_build_var(environ_cp, 'TF_NEED_GCP', 'Google Cloud Platform', - 'with_gcp_support', False) + 'with_gcp_support', True, 'gcp') set_build_var(environ_cp, 'TF_NEED_HDFS', 'Hadoop File System', - 'with_hdfs_support', False) + 'with_hdfs_support', True, 'hdfs') + set_build_var(environ_cp, 'TF_NEED_S3', 'Amazon S3 File System', + 'with_s3_support', True, 's3') set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support', - False) + False, 'xla') set_build_var(environ_cp, 'TF_NEED_GDR', 'GDR', 'with_gdr_support', - False) + False, 'gdr') set_build_var(environ_cp, 'TF_NEED_VERBS', 'VERBS', 'with_verbs_support', - False) + False, 'verbs') - set_action_env_var(environ_cp, 'TF_NEED_OPENCL', 'OpenCL', False) - if environ_cp.get('TF_NEED_OPENCL') == '1': + set_action_env_var(environ_cp, 'TF_NEED_OPENCL_SYCL', 'OpenCL SYCL', False) + if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1': set_host_cxx_compiler(environ_cp) set_host_c_compiler(environ_cp) - set_computecpp_toolkit_path(environ_cp) + set_action_env_var(environ_cp, 'TF_NEED_COMPUTECPP', 'ComputeCPP', True) + if environ_cp.get('TF_NEED_COMPUTECPP') == '1': + set_computecpp_toolkit_path(environ_cp) + else: + set_trisycl_include_dir(environ_cp) set_action_env_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False) - if environ_cp.get('TF_NEED_CUDA') == '1': + if (environ_cp.get('TF_NEED_CUDA') == '1' and + 'TF_CUDA_CONFIG_REPO' not in environ_cp): set_tf_cuda_version(environ_cp) - set_tf_cunn_version(environ_cp) + set_tf_cudnn_version(environ_cp) set_tf_cuda_compute_capabilities(environ_cp) set_tf_cuda_clang(environ_cp) if environ_cp.get('TF_CUDA_CLANG') == '1': - # Set up which clang we should use as the cuda / host compiler. - set_clang_cuda_compiler_path(environ_cp) + if not is_windows(): + # Ask if we want to download clang release while building. + set_tf_download_clang(environ_cp) + else: + # We use bazel's generated crosstool on Windows and there is no + # way to provide downloaded toolchain for that yet. + # TODO(ibiryukov): Investigate using clang as a cuda compiler on + # Windows. + environ_cp['TF_DOWNLOAD_CLANG'] = '0' + + if environ_cp.get('TF_DOWNLOAD_CLANG') != '1': + # Set up which clang we should use as the cuda / host compiler. + set_clang_cuda_compiler_path(environ_cp) else: # Set up which gcc nvcc should use as the host compiler # No need to set this on Windows @@ -1013,8 +1311,28 @@ def main(): set_mpi_home(environ_cp) set_other_mpi_vars(environ_cp) + set_grpc_build_flags() set_cc_opt_flags(environ_cp) set_mkl() + set_monolithic() + set_windows_build_flags() + create_android_bazelrc_configs() + + if workspace_has_any_android_rule(): + print('The WORKSPACE file has at least one of ["android_sdk_repository", ' + '"android_ndk_repository"] already set. Will not ask to help ' + 'configure the WORKSPACE. Please delete the existing rules to ' + 'activate the helper.\n') + else: + if get_var( + environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace', + False, + ('Would you like to interactively configure ./WORKSPACE for ' + 'Android builds?'), + 'Searching for NDK and SDK installations.', + 'Not configuring the WORKSPACE for Android builds.'): + create_android_ndk_rule(environ_cp) + create_android_sdk_rule(environ_cp) if __name__ == '__main__': diff --git a/tensorflow/BUILD b/tensorflow/BUILD index 4dc84889137bed48a0145cf34757d1970708a5a7..6f7e54a90a624d371f1311fcef1807eb90ed36ff 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -9,8 +9,17 @@ licenses(["notice"]) # Apache 2.0 exports_files([ "LICENSE", "ACKNOWLEDGMENTS", + # The leakr files are used by //third_party/cloud_tpu. + "leakr_badwords.dic", + "leakr_badfiles.dic", ]) +load("//tensorflow:tensorflow.bzl", "tf_cc_shared_object") +load( + "//tensorflow/core:platform/default/build_config.bzl", + "tf_additional_binary_deps", +) + # Config setting for determining if we are building for Android. config_setting( name = "android", @@ -45,6 +54,15 @@ config_setting( visibility = ["//visibility:public"], ) +config_setting( + name = "raspberry_pi_armeabi", + values = { + "crosstool_top": "@local_config_arm_compiler//:toolchain", + "cpu": "armeabi", + }, + visibility = ["//visibility:public"], +) + config_setting( name = "android_arm", values = { @@ -101,7 +119,7 @@ config_setting( config_setting( name = "no_tensorflow_py_deps", - values = {"define": "no_tensorflow_py_deps=true"}, + define_values = {"no_tensorflow_py_deps": "true"}, visibility = ["//visibility:public"], ) @@ -111,6 +129,15 @@ config_setting( visibility = ["//visibility:public"], ) +config_setting( + name = "ios_x86_64", + values = { + "crosstool_top": "//tools/osx/crosstool:crosstool", + "cpu": "ios_x86_64", + }, + visibility = ["//visibility:public"], +) + config_setting( name = "linux_x86_64", values = {"cpu": "k8"}, @@ -148,52 +175,189 @@ config_setting( # TODO(jhseu): Enable on other platforms other than Linux. config_setting( name = "with_jemalloc_linux_x86_64", - values = { - "cpu": "k8", - "define": "with_jemalloc=true", - }, + define_values = {"with_jemalloc": "true"}, + values = {"cpu": "k8"}, visibility = ["//visibility:public"], ) config_setting( name = "with_jemalloc_linux_ppc64le", - values = { - "cpu": "ppc", - "define": "with_jemalloc=true", - }, + define_values = {"with_jemalloc": "true"}, + values = {"cpu": "ppc"}, + visibility = ["//visibility:public"], +) + +config_setting( + name = "with_default_optimizations", + define_values = {"with_default_optimizations": "true"}, visibility = ["//visibility:public"], ) config_setting( name = "with_gcp_support", - values = {"define": "with_gcp_support=true"}, + define_values = {"with_gcp_support": "true"}, visibility = ["//visibility:public"], ) config_setting( name = "with_hdfs_support", - values = {"define": "with_hdfs_support=true"}, + define_values = {"with_hdfs_support": "true"}, + visibility = ["//visibility:public"], +) + +config_setting( + name = "with_s3_support", + define_values = {"with_s3_support": "true"}, + visibility = ["//visibility:public"], +) + +# Crosses between platforms and file system libraries not supported on those +# platforms due to limitations in nested select() statements. +config_setting( + name = "with_gcp_support_windows_override", + define_values = {"with_gcp_support": "true"}, + values = {"cpu": "x64_windows"}, + visibility = ["//visibility:public"], +) + +config_setting( + name = "with_hdfs_support_windows_override", + define_values = {"with_hdfs_support": "true"}, + values = {"cpu": "x64_windows"}, + visibility = ["//visibility:public"], +) + +config_setting( + name = "with_s3_support_windows_override", + define_values = {"with_s3_support": "true"}, + values = {"cpu": "x64_windows"}, + visibility = ["//visibility:public"], +) + +config_setting( + name = "with_gcp_support_android_override", + define_values = {"with_gcp_support": "true"}, + values = {"crosstool_top": "//external:android/crosstool"}, + visibility = ["//visibility:public"], +) + +config_setting( + name = "with_hdfs_support_android_override", + define_values = {"with_hdfs_support": "true"}, + values = {"crosstool_top": "//external:android/crosstool"}, + visibility = ["//visibility:public"], +) + +config_setting( + name = "with_s3_support_android_override", + define_values = {"with_s3_support": "true"}, + values = {"crosstool_top": "//external:android/crosstool"}, + visibility = ["//visibility:public"], +) + +config_setting( + name = "with_gcp_support_ios_override", + define_values = {"with_gcp_support": "true"}, + values = {"crosstool_top": "//tools/osx/crosstool:crosstool"}, + visibility = ["//visibility:public"], +) + +config_setting( + name = "with_hdfs_support_ios_override", + define_values = {"with_hdfs_support": "true"}, + values = {"crosstool_top": "//tools/osx/crosstool:crosstool"}, + visibility = ["//visibility:public"], +) + +config_setting( + name = "with_s3_support_ios_override", + define_values = {"with_s3_support": "true"}, + values = {"crosstool_top": "//tools/osx/crosstool:crosstool"}, visibility = ["//visibility:public"], ) config_setting( name = "with_xla_support", - values = {"define": "with_xla_support=true"}, + define_values = {"with_xla_support": "true"}, visibility = ["//visibility:public"], ) config_setting( name = "with_gdr_support", - values = {"define": "with_gdr_support=true"}, + define_values = {"with_gdr_support": "true"}, visibility = ["//visibility:public"], ) config_setting( name = "with_verbs_support", - values = {"define": "with_verbs_support=true"}, + define_values = {"with_verbs_support": "true"}, + visibility = ["//visibility:public"], +) + +# Crosses between framework_shared_object and a bunch of other configurations +# due to limitations in nested select() statements. +config_setting( + name = "framework_shared_object", + define_values = { + "framework_shared_object": "true", + }, + visibility = ["//visibility:public"], +) + +config_setting( + name = "with_jemalloc_linux_x86_64_dynamic", + define_values = { + "with_jemalloc": "true", + "framework_shared_object": "true", + }, + values = { + "cpu": "k8", + }, visibility = ["//visibility:public"], ) +config_setting( + name = "with_jemalloc_linux_ppc64le_dynamic", + define_values = { + "with_jemalloc": "true", + "framework_shared_object": "true", + }, + values = { + "cpu": "ppc", + }, + visibility = ["//visibility:public"], +) + +config_setting( + name = "using_cuda_clang", + define_values = { + "using_cuda_clang": "true", + }, +) + +config_setting( + name = "using_cuda_clang_with_dynamic_build", + define_values = { + "using_cuda_clang": "true", + "framework_shared_object": "true", + }, +) + +config_setting( + name = "using_cuda_nvcc", + define_values = { + "using_cuda_nvcc": "true", + }, +) + +config_setting( + name = "using_cuda_nvcc_with_dynamic_build", + define_values = { + "using_cuda_nvcc": "true", + "framework_shared_object": "true", + }, +) + config_setting( name = "with_mpi_support", values = {"define": "with_mpi_support=true"}, @@ -203,8 +367,9 @@ config_setting( package_group( name = "internal", packages = [ - "//learning/protonn/llgtm/...", + "//learning/meta_rank/...", "//tensorflow/...", + "//tensorflow_fold/llgtm/...", ], ) @@ -243,24 +408,29 @@ filegroup( "//tensorflow/compiler/jit/kernels:all_files", "//tensorflow/compiler/jit/legacy_flags:all_files", "//tensorflow/compiler/jit/ops:all_files", - "//tensorflow/compiler/plugin/executor:all_files", + "//tensorflow/compiler/plugin:all_files", "//tensorflow/compiler/tests:all_files", "//tensorflow/compiler/tf2xla:all_files", "//tensorflow/compiler/tf2xla/cc:all_files", "//tensorflow/compiler/tf2xla/kernels:all_files", + "//tensorflow/compiler/tf2xla/lib:all_files", "//tensorflow/compiler/tf2xla/ops:all_files", "//tensorflow/compiler/xla:all_files", "//tensorflow/compiler/xla/client:all_files", "//tensorflow/compiler/xla/client/lib:all_files", "//tensorflow/compiler/xla/legacy_flags:all_files", + "//tensorflow/compiler/xla/python:all_files", "//tensorflow/compiler/xla/service:all_files", "//tensorflow/compiler/xla/service/cpu:all_files", "//tensorflow/compiler/xla/service/gpu:all_files", "//tensorflow/compiler/xla/service/gpu/llvm_gpu_backend:all_files", + "//tensorflow/compiler/xla/service/interpreter:all_files", "//tensorflow/compiler/xla/service/llvm_ir:all_files", "//tensorflow/compiler/xla/tests:all_files", "//tensorflow/compiler/xla/tools:all_files", + "//tensorflow/compiler/xla/tools/parser:all_files", "//tensorflow/contrib:all_files", + "//tensorflow/contrib/all_reduce:all_files", "//tensorflow/contrib/android:all_files", "//tensorflow/contrib/batching:all_files", "//tensorflow/contrib/batching/kernels:all_files", @@ -280,46 +450,88 @@ filegroup( "//tensorflow/contrib/crf:all_files", "//tensorflow/contrib/cudnn_rnn:all_files", "//tensorflow/contrib/data:all_files", - "//tensorflow/contrib/data/python/framework:all_files", + "//tensorflow/contrib/data/kernels:all_files", "//tensorflow/contrib/data/python/kernel_tests:all_files", "//tensorflow/contrib/data/python/ops:all_files", - "//tensorflow/contrib/data/python/util:all_files", "//tensorflow/contrib/decision_trees/proto:all_files", + "//tensorflow/contrib/deprecated:all_files", "//tensorflow/contrib/distributions:all_files", + "//tensorflow/contrib/eager/proto:all_files", "//tensorflow/contrib/eager/python:all_files", + "//tensorflow/contrib/estimator:all_files", "//tensorflow/contrib/factorization:all_files", + "//tensorflow/contrib/factorization/examples:all_files", "//tensorflow/contrib/factorization/kernels:all_files", "//tensorflow/contrib/ffmpeg:all_files", "//tensorflow/contrib/ffmpeg/default:all_files", "//tensorflow/contrib/framework:all_files", "//tensorflow/contrib/fused_conv:all_files", + "//tensorflow/contrib/gan:all_files", + "//tensorflow/contrib/gdr:all_files", "//tensorflow/contrib/graph_editor:all_files", "//tensorflow/contrib/grid_rnn:all_files", "//tensorflow/contrib/hooks:all_files", + "//tensorflow/contrib/hvx/clock_cycle_profiling:all_files", "//tensorflow/contrib/hvx/hvx_ops_support_checker:all_files", "//tensorflow/contrib/image:all_files", - "//tensorflow/contrib/imperative:all_files", "//tensorflow/contrib/input_pipeline:all_files", "//tensorflow/contrib/input_pipeline/kernels:all_files", "//tensorflow/contrib/integrate:all_files", "//tensorflow/contrib/keras:all_files", "//tensorflow/contrib/kernel_methods:all_files", + "//tensorflow/contrib/kfac:all_files", + "//tensorflow/contrib/kfac/examples:all_files", + "//tensorflow/contrib/kfac/examples/tests:all_files", + "//tensorflow/contrib/kfac/python/kernel_tests:all_files", + "//tensorflow/contrib/kfac/python/ops:all_files", "//tensorflow/contrib/labeled_tensor:all_files", "//tensorflow/contrib/layers:all_files", "//tensorflow/contrib/layers/kernels:all_files", "//tensorflow/contrib/learn:all_files", "//tensorflow/contrib/learn/python/learn/datasets:all_files", + "//tensorflow/contrib/legacy_seq2seq:all_files", + "//tensorflow/contrib/libsvm:all_files", "//tensorflow/contrib/linalg:all_files", "//tensorflow/contrib/linear_optimizer:all_files", + "//tensorflow/contrib/lite:all_files", + "//tensorflow/contrib/lite/java:all_files", + "//tensorflow/contrib/lite/java/demo/app/src/main:all_files", + "//tensorflow/contrib/lite/java/demo/app/src/main/assets:all_files", + "//tensorflow/contrib/lite/java/src/main/native:all_files", + "//tensorflow/contrib/lite/java/src/testhelper/java/org/tensorflow/lite:all_files", + "//tensorflow/contrib/lite/kernels:all_files", + "//tensorflow/contrib/lite/kernels/internal:all_files", + "//tensorflow/contrib/lite/models/smartreply:all_files", + "//tensorflow/contrib/lite/nnapi:all_files", + "//tensorflow/contrib/lite/python:all_files", + "//tensorflow/contrib/lite/schema:all_files", + "//tensorflow/contrib/lite/testing:all_files", + "//tensorflow/contrib/lite/toco:all_files", + "//tensorflow/contrib/lite/toco/graph_transformations/tests:all_files", + "//tensorflow/contrib/lite/toco/python:all_files", + "//tensorflow/contrib/lite/toco/tensorflow_graph_matching:all_files", + "//tensorflow/contrib/lite/toco/tflite:all_files", + "//tensorflow/contrib/lite/tools:all_files", "//tensorflow/contrib/lookup:all_files", "//tensorflow/contrib/losses:all_files", + "//tensorflow/contrib/makefile:all_files", + "//tensorflow/contrib/memory_stats:all_files", "//tensorflow/contrib/meta_graph_transform:all_files", "//tensorflow/contrib/metrics:all_files", + "//tensorflow/contrib/model_pruning:all_files", + "//tensorflow/contrib/model_pruning/examples/cifar10:all_files", + "//tensorflow/contrib/mpi:all_files", + "//tensorflow/contrib/mpi_collectives:all_files", + "//tensorflow/contrib/nccl:all_files", "//tensorflow/contrib/ndlstm:all_files", "//tensorflow/contrib/nearest_neighbor:all_files", "//tensorflow/contrib/nn:all_files", "//tensorflow/contrib/opt:all_files", + "//tensorflow/contrib/periodic_resample:all_files", "//tensorflow/contrib/predictor:all_files", + "//tensorflow/contrib/quantization:all_files", + "//tensorflow/contrib/quantize:all_files", + "//tensorflow/contrib/receptive_field:all_files", "//tensorflow/contrib/reduce_slice_ops:all_files", "//tensorflow/contrib/remote_fused_graph/pylib:all_files", "//tensorflow/contrib/resampler:all_files", @@ -339,11 +551,13 @@ filegroup( "//tensorflow/contrib/staging:all_files", "//tensorflow/contrib/stat_summarizer:all_files", "//tensorflow/contrib/stateless:all_files", + "//tensorflow/contrib/summary:all_files", "//tensorflow/contrib/tensor_forest:all_files", "//tensorflow/contrib/tensor_forest/hybrid:all_files", "//tensorflow/contrib/tensor_forest/kernels/v4:all_files", "//tensorflow/contrib/tensor_forest/proto:all_files", "//tensorflow/contrib/tensorboard:all_files", + "//tensorflow/contrib/tensorboard/db:all_files", "//tensorflow/contrib/testing:all_files", "//tensorflow/contrib/text:all_files", "//tensorflow/contrib/tfprof:all_files", @@ -352,10 +566,11 @@ filegroup( "//tensorflow/contrib/timeseries/python/timeseries:all_files", "//tensorflow/contrib/timeseries/python/timeseries/state_space_models:all_files", "//tensorflow/contrib/tpu:all_files", + "//tensorflow/contrib/tpu/profiler:all_files", + "//tensorflow/contrib/tpu/proto:all_files", "//tensorflow/contrib/training:all_files", "//tensorflow/contrib/util:all_files", "//tensorflow/contrib/verbs:all_files", - "//tensorflow/contrib/xla_tf_graph:all_files", "//tensorflow/core:all_files", "//tensorflow/core/debug:all_files", "//tensorflow/core/distributed_runtime:all_files", @@ -367,18 +582,23 @@ filegroup( "//tensorflow/core/grappler/optimizers:all_files", "//tensorflow/core/grappler/utils:all_files", "//tensorflow/core/kernels:all_files", + "//tensorflow/core/kernels/data:all_files", + "//tensorflow/core/kernels/data/sql:all_files", "//tensorflow/core/kernels/fuzzing:all_files", "//tensorflow/core/kernels/hexagon:all_files", "//tensorflow/core/kernels/neon:all_files", + "//tensorflow/core/lib/db:all_files", "//tensorflow/core/ops/compat:all_files", "//tensorflow/core/platform/cloud:all_files", "//tensorflow/core/platform/default/build_config:all_files", "//tensorflow/core/platform/hadoop:all_files", + "//tensorflow/core/platform/s3:all_files", "//tensorflow/core/profiler:all_files", "//tensorflow/core/profiler/internal:all_files", "//tensorflow/core/profiler/internal/advisor:all_files", "//tensorflow/core/util/ctc:all_files", "//tensorflow/core/util/tensor_bundle:all_files", + "//tensorflow/examples/adding_an_op:all_files", "//tensorflow/examples/android:all_files", "//tensorflow/examples/benchmark:all_files", "//tensorflow/examples/get_started/regression:all_files", @@ -386,10 +606,13 @@ filegroup( "//tensorflow/examples/image_retraining:all_files", "//tensorflow/examples/label_image:all_files", "//tensorflow/examples/learn:all_files", + "//tensorflow/examples/multibox_detector:all_files", "//tensorflow/examples/saved_model:all_files", "//tensorflow/examples/speech_commands:all_files", "//tensorflow/examples/tutorials/estimators:all_files", + "//tensorflow/examples/tutorials/layers:all_files", "//tensorflow/examples/tutorials/mnist:all_files", + "//tensorflow/examples/tutorials/monitors:all_files", "//tensorflow/examples/tutorials/word2vec:all_files", "//tensorflow/examples/wav_to_spectrogram:all_files", "//tensorflow/go:all_files", @@ -397,20 +620,33 @@ filegroup( "//tensorflow/java/src/main/java/org/tensorflow/examples:all_files", "//tensorflow/java/src/main/native:all_files", "//tensorflow/python:all_files", + "//tensorflow/python/data:all_files", + "//tensorflow/python/data/kernel_tests:all_files", + "//tensorflow/python/data/ops:all_files", + "//tensorflow/python/data/util:all_files", "//tensorflow/python/debug:all_files", "//tensorflow/python/eager:all_files", "//tensorflow/python/estimator:all_files", "//tensorflow/python/feature_column:all_files", + "//tensorflow/python/keras:all_files", "//tensorflow/python/kernel_tests:all_files", "//tensorflow/python/kernel_tests/distributions:all_files", + "//tensorflow/python/kernel_tests/linalg:all_files", + "//tensorflow/python/kernel_tests/random:all_files", "//tensorflow/python/ops/distributions:all_files", + "//tensorflow/python/ops/linalg:all_files", + "//tensorflow/python/ops/losses:all_files", "//tensorflow/python/profiler:all_files", "//tensorflow/python/profiler/internal:all_files", "//tensorflow/python/saved_model:all_files", "//tensorflow/python/tools:all_files", + "//tensorflow/tools/api/generator:all_files", "//tensorflow/tools/api/golden:all_files", "//tensorflow/tools/api/lib:all_files", "//tensorflow/tools/api/tests:all_files", + "//tensorflow/tools/benchmark:all_files", + "//tensorflow/tools/build_info:all_files", + "//tensorflow/tools/ci_build/gpu_build:all_files", "//tensorflow/tools/common:all_files", "//tensorflow/tools/compatibility:all_files", "//tensorflow/tools/dist_test/server:all_files", @@ -418,16 +654,22 @@ filegroup( "//tensorflow/tools/docker/notebooks:all_files", "//tensorflow/tools/docs:all_files", "//tensorflow/tools/git:all_files", + "//tensorflow/tools/graph_transforms:all_files", "//tensorflow/tools/mlpbtxt:all_files", "//tensorflow/tools/proto_text:all_files", "//tensorflow/tools/quantization:all_files", "//tensorflow/tools/test:all_files", "//tensorflow/user_ops:all_files", + "//third_party/eigen3:all_files", + "//third_party/fft2d:all_files", + "//third_party/flatbuffers:all_files", "//third_party/hadoop:all_files", + "//third_party/mpi:all_files", + "//third_party/mpi_collectives:all_files", "//third_party/sycl:all_files", "//third_party/sycl/sycl:all_files", ], - visibility = [":__subpackages__"], + visibility = ["//visibility:public"], ) load( @@ -449,15 +691,65 @@ filegroup( data = glob(["docs_src/**/*.md"]), ) +# A shared object which includes registration mechanisms for ops and +# kernels. Does not include the implementations of any ops or kernels. Instead, +# the library which loads libtensorflow_framework.so +# (e.g. _pywrap_tensorflow_internal.so for Python, libtensorflow.so for the C +# API) is responsible for registering ops with libtensorflow_framework.so. In +# addition to this core set of ops, user libraries which are loaded (via +# TF_LoadLibrary/tf.load_op_library) register their ops and kernels with this +# shared object directly. +# +# For example, from Python tf.load_op_library loads a custom op library (via +# dlopen() on Linux), the library finds libtensorflow_framework.so (no +# filesystem search takes place, since libtensorflow_framework.so has already +# been loaded by pywrap_tensorflow) and registers its ops and kernels via +# REGISTER_OP and REGISTER_KERNEL_BUILDER (which use symbols from +# libtensorflow_framework.so), and pywrap_tensorflow can then use these +# ops. Since other languages use the same libtensorflow_framework.so, op +# libraries are language agnostic. +# +# This shared object is not used unless framework_shared_object=true (set in the +# configure script unconditionally); otherwise if it is false or undefined, the +# build is static and TensorFlow symbols (in Python only) are loaded into the +# global symbol table in order to support op registration. This means that +# projects building with Bazel and importing TensorFlow as a dependency will not +# depend on libtensorflow_framework.so unless they opt in. +tf_cc_shared_object( + name = "libtensorflow_framework.so", + framework_so = [], + linkstatic = 1, + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/core:framework_internal_impl", + "//tensorflow/core:lib_internal_impl", + "//tensorflow/core:core_cpu_impl", + "//tensorflow/stream_executor:stream_executor_impl", + "//tensorflow/core:gpu_runtime_impl", + ] + tf_additional_binary_deps(), +) + # ------------------------------------------- # New rules should be added above this target. # ------------------------------------------- -cc_binary( + +# TensorFlow uses several libraries that may also be used by applications +# linking against the C and C++ APIs (such as libjpeg). When we create +# the shared library, only export the core TF API functions to avoid +# causing library conflicts (e.g., those reported in github issue 1924). +# On Linux, tell the linker (-Wl,