Skip to content

GitLab

Commit 510e763c authored by Mark Daoust's avatar Mark Daoust Committed by A. Unique TensorFlower
Browse files

Internal change 

PiperOrigin-RevId: 440152953
parent 92266d5d
Show whitespace changes
Inline Side-by-side
Showing with 79 additions and 288 deletions
official/utils/docs/build_api_docs_lib.py deleted 100644 → 0
View file @ 92266d5d
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
r"""Common library for API docs builder."""
import tensorflow as tf
from tensorflow_docs.api_generator import doc_controls
def hide_module_model_and_layer_methods():
"""Hide methods and properties defined in the base classes of Keras layers.
We hide all methods and properties of the base classes, except:
- `__init__` is always documented.
- `call` is always documented, as it can carry important information for
complex layers.
"""
module_contents = list(tf.Module.__dict__.items())
model_contents = list(tf.keras.Model.__dict__.items())
layer_contents = list(tf.keras.layers.Layer.__dict__.items())
for name, obj in module_contents + layer_contents + model_contents:
if name == '__init__':
# Always document __init__.
continue
if name == 'call':
# Always document `call`.
if hasattr(obj, doc_controls._FOR_SUBCLASS_IMPLEMENTERS): # pylint: disable=protected-access
delattr(obj, doc_controls._FOR_SUBCLASS_IMPLEMENTERS) # pylint: disable=protected-access
continue
# Otherwise, exclude from documentation.
if isinstance(obj, property):
obj = obj.fget
if isinstance(obj, (staticmethod, classmethod)):
obj = obj.__func__
try:
doc_controls.do_not_doc_in_subclasses(obj)
except AttributeError:
pass
official/utils/docs/build_nlp_api_docs.py deleted 100644 → 0
View file @ 92266d5d
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
r"""Tool to generate api_docs for tensorflow_models/official library.
Example:
$> pip install -U git+https://github.com/tensorflow/docs
$> python build_nlp_api_docs.py --output_dir=/tmp/api_docs
"""
import pathlib
from absl import app
from absl import flags
from absl import logging
from tensorflow_docs.api_generator import generate_lib
from tensorflow_docs.api_generator import public_api
import tensorflow_models as tfm
import build_api_docs_lib
FLAGS = flags.FLAGS
flags.DEFINE_string('output_dir', None, 'Where to write the resulting docs to.')
flags.DEFINE_string(
'code_url_prefix',
'https://github.com/tensorflow/models/blob/master/tensorflow_models/nlp',
'The url prefix for links to code.')
flags.DEFINE_bool('search_hints', True,
'Include metadata search hints in the generated files')
flags.DEFINE_string('site_path', '/api_docs/python',
'Path prefix in the _toc.yaml')
PROJECT_SHORT_NAME = 'tfm.nlp'
PROJECT_FULL_NAME = 'TensorFlow Official Models - NLP Modeling Library'
def custom_filter(path, parent, children):
if len(path) == 1:
# Don't filter the contents of the top level `tfm.vision` package.
return children
else:
return public_api.explicit_package_contents_filter(path, parent, children)
def gen_api_docs(code_url_prefix, site_path, output_dir, project_short_name,
project_full_name, search_hints):
"""Generates api docs for the tensorflow docs package."""
build_api_docs_lib.hide_module_model_and_layer_methods()
del tfm.nlp.layers.MultiHeadAttention
del tfm.nlp.layers.EinsumDense
url_parts = code_url_prefix.strip('/').split('/')
url_parts = url_parts[:url_parts.index('tensorflow_models')]
url_parts.append('official')
official_url_prefix = '/'.join(url_parts)
nlp_base_dir = pathlib.Path(tfm.nlp.__file__).parent
# The `layers` submodule (and others) are actually defined in the `official`
# package. Find the path to `official`.
official_base_dir = [
p for p in pathlib.Path(tfm.vision.layers.__file__).parents
if p.name == 'official'
][0]
doc_generator = generate_lib.DocGenerator(
root_title=project_full_name,
py_modules=[(project_short_name, tfm.nlp)],
base_dir=[nlp_base_dir, official_base_dir],
code_url_prefix=[
code_url_prefix,
official_url_prefix,
],
search_hints=search_hints,
site_path=site_path,
callbacks=[custom_filter],
)
doc_generator.build(output_dir)
logging.info('Output docs to: %s', output_dir)
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
gen_api_docs(
code_url_prefix=FLAGS.code_url_prefix,
site_path=FLAGS.site_path,
output_dir=FLAGS.output_dir,
project_short_name=PROJECT_SHORT_NAME,
project_full_name=PROJECT_FULL_NAME,
search_hints=FLAGS.search_hints)
if __name__ == '__main__':
flags.mark_flag_as_required('output_dir')
app.run(main)
official/utils/docs/build_orbit_api_docs.py
View file @ 510e763c
...@@ -25,11 +25,11 @@ from absl import logging ...@@ -25,11 +25,11 @@ from absl import logging
import orbit import orbit
import tensorflow as tf
from tensorflow_docs.api_generator import doc_controls
from tensorflow_docs.api_generator import generate_lib from tensorflow_docs.api_generator import generate_lib
from tensorflow_docs.api_generator import public_api from tensorflow_docs.api_generator import public_api
from official.utils.docs import build_api_docs_lib
FLAGS = flags.FLAGS FLAGS = flags.FLAGS
flags.DEFINE_string('output_dir', None, 'Where to write the resulting docs to.') flags.DEFINE_string('output_dir', None, 'Where to write the resulting docs to.')
...@@ -48,10 +48,45 @@ PROJECT_SHORT_NAME = 'orbit' ...@@ -48,10 +48,45 @@ PROJECT_SHORT_NAME = 'orbit'
PROJECT_FULL_NAME = 'Orbit' PROJECT_FULL_NAME = 'Orbit'
def hide_module_model_and_layer_methods():
"""Hide methods and properties defined in the base classes of Keras layers.
We hide all methods and properties of the base classes, except:
- `__init__` is always documented.
- `call` is always documented, as it can carry important information for
complex layers.
"""
module_contents = list(tf.Module.__dict__.items())
model_contents = list(tf.keras.Model.__dict__.items())
layer_contents = list(tf.keras.layers.Layer.__dict__.items())
for name, obj in module_contents + layer_contents + model_contents:
if name == '__init__':
# Always document __init__.
continue
if name == 'call':
# Always document `call`.
if hasattr(obj, doc_controls._FOR_SUBCLASS_IMPLEMENTERS): # pylint: disable=protected-access
delattr(obj, doc_controls._FOR_SUBCLASS_IMPLEMENTERS) # pylint: disable=protected-access
continue
# Otherwise, exclude from documentation.
if isinstance(obj, property):
obj = obj.fget
if isinstance(obj, (staticmethod, classmethod)):
obj = obj.__func__
try:
doc_controls.do_not_doc_in_subclasses(obj)
except AttributeError:
pass
def gen_api_docs(code_url_prefix, site_path, output_dir, project_short_name, def gen_api_docs(code_url_prefix, site_path, output_dir, project_short_name,
project_full_name, search_hints): project_full_name, search_hints):
"""Generates api docs for the tensorflow docs package.""" """Generates api docs for the tensorflow docs package."""
build_api_docs_lib.hide_module_model_and_layer_methods()
doc_generator = generate_lib.DocGenerator( doc_generator = generate_lib.DocGenerator(
root_title=project_full_name, root_title=project_full_name,
......
official/utils/docs/build_all_api_docs.py → official/utils/docs/build_tfm_api_docs.py
View file @ 510e763c
...@@ -25,11 +25,13 @@ import pathlib ...@@ -25,11 +25,13 @@ import pathlib
from absl import app from absl import app
from absl import flags from absl import flags
from absl import logging from absl import logging
import tensorflow as tf
from tensorflow_docs.api_generator import doc_controls
from tensorflow_docs.api_generator import generate_lib from tensorflow_docs.api_generator import generate_lib
from tensorflow_docs.api_generator import public_api from tensorflow_docs.api_generator import public_api
import tensorflow_models as tfm import tensorflow_models as tfm
from official.utils.docs import build_api_docs_lib
FLAGS = flags.FLAGS FLAGS = flags.FLAGS
...@@ -47,7 +49,43 @@ flags.DEFINE_string('site_path', '/api_docs/python', ...@@ -47,7 +49,43 @@ flags.DEFINE_string('site_path', '/api_docs/python',
PROJECT_SHORT_NAME = 'tfm' PROJECT_SHORT_NAME = 'tfm'
PROJECT_FULL_NAME = 'TensorFlow Official Models - Modeling Library' PROJECT_FULL_NAME = 'TensorFlow Modeling Library'
def hide_module_model_and_layer_methods():
"""Hide methods and properties defined in the base classes of Keras layers.
We hide all methods and properties of the base classes, except:
- `__init__` is always documented.
- `call` is always documented, as it can carry important information for
complex layers.
"""
module_contents = list(tf.Module.__dict__.items())
model_contents = list(tf.keras.Model.__dict__.items())
layer_contents = list(tf.keras.layers.Layer.__dict__.items())
for name, obj in module_contents + layer_contents + model_contents:
if name == '__init__':
# Always document __init__.
continue
if name == 'call':
# Always document `call`.
if hasattr(obj, doc_controls._FOR_SUBCLASS_IMPLEMENTERS): # pylint: disable=protected-access
delattr(obj, doc_controls._FOR_SUBCLASS_IMPLEMENTERS) # pylint: disable=protected-access
continue
# Otherwise, exclude from documentation.
if isinstance(obj, property):
obj = obj.fget
if isinstance(obj, (staticmethod, classmethod)):
obj = obj.__func__
try:
doc_controls.do_not_doc_in_subclasses(obj)
except AttributeError:
pass
def custom_filter(path, parent, children): def custom_filter(path, parent, children):
...@@ -61,7 +99,7 @@ def custom_filter(path, parent, children): ...@@ -61,7 +99,7 @@ def custom_filter(path, parent, children):
def gen_api_docs(code_url_prefix, site_path, output_dir, project_short_name, def gen_api_docs(code_url_prefix, site_path, output_dir, project_short_name,
project_full_name, search_hints): project_full_name, search_hints):
"""Generates api docs for the tensorflow docs package.""" """Generates api docs for the tensorflow docs package."""
build_api_docs_lib.hide_module_model_and_layer_methods() hide_module_model_and_layer_methods()
del tfm.nlp.layers.MultiHeadAttention del tfm.nlp.layers.MultiHeadAttention
del tfm.nlp.layers.EinsumDense del tfm.nlp.layers.EinsumDense
......
official/utils/docs/build_vision_api_docs.py deleted 100644 → 0
View file @ 92266d5d
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
r"""Tool to generate api_docs for tensorflow_models/official library.
Example:
$> pip install -U git+https://github.com/tensorflow/docs
$> python build_vision_api_docs.py --output_dir=/tmp/api_docs
"""
import pathlib
from absl import app
from absl import flags
from absl import logging
from tensorflow_docs.api_generator import generate_lib
from tensorflow_docs.api_generator import public_api
import tensorflow_models as tfm
import build_api_docs_lib
FLAGS = flags.FLAGS
flags.DEFINE_string('output_dir', None, 'Where to write the resulting docs to.')
flags.DEFINE_string(
'code_url_prefix',
'https://github.com/tensorflow/models/blob/master/tensorflow_models/vision',
'The url prefix for links to code.')
flags.DEFINE_bool('search_hints', True,
'Include metadata search hints in the generated files')
flags.DEFINE_string('site_path', 'tfvision/api_docs/python',
'Path prefix in the _toc.yaml')
PROJECT_SHORT_NAME = 'tfm.vision'
PROJECT_FULL_NAME = 'TensorFlow Official Models - Vision Modeling Library'
def custom_filter(path, parent, children):
if len(path) == 1:
# Don't filter the contents of the top level `tfm.vision` package.
return children
else:
return public_api.explicit_package_contents_filter(path, parent, children)
def gen_api_docs(code_url_prefix, site_path, output_dir, project_short_name,
project_full_name, search_hints):
"""Generates api docs for the tensorflow docs package."""
build_api_docs_lib.hide_module_model_and_layer_methods()
url_parts = code_url_prefix.strip('/').split('/')
url_parts = url_parts[:url_parts.index('tensorflow_models')]
url_parts.append('official')
official_url_prefix = '/'.join(url_parts)
vision_base_dir = pathlib.Path(tfm.vision.__file__).parent
# The `layers` submodule (and others) are actually defined in the `official`
# package. Find the path to `official`.
official_base_dir = [
p for p in pathlib.Path(tfm.vision.layers.__file__).parents
if p.name == 'official'
][0]
doc_generator = generate_lib.DocGenerator(
root_title=project_full_name,
py_modules=[(project_short_name, tfm.vision)],
base_dir=[
vision_base_dir,
official_base_dir,
],
code_url_prefix=[code_url_prefix, official_url_prefix],
search_hints=search_hints,
site_path=site_path,
callbacks=[custom_filter],
)
doc_generator.build(output_dir)
logging.info('Output docs to: %s', output_dir)
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
gen_api_docs(
code_url_prefix=FLAGS.code_url_prefix,
site_path=FLAGS.site_path,
output_dir=FLAGS.output_dir,
project_short_name=PROJECT_SHORT_NAME,
project_full_name=PROJECT_FULL_NAME,
search_hints=FLAGS.search_hints)
if __name__ == '__main__':
flags.mark_flag_as_required('output_dir')
app.run(main)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment