diff --git a/.gitignore b/.gitignore index 2a5279e..12ce418 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,15 @@ -*.egg-info -.mypy_cache -.pytest_cache __pycache__ -dist -docs/_build -venv -build \ No newline at end of file +/build +/dist +/docs/_build +/.venv +/*.egg-info +/.ipynb_checkpoints +/.mypy_cache +/.pytest_cache +/.vscode +/.tool-versions +/test_* +/*.log +.idea +.DS_Store \ No newline at end of file diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..70a3958 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,10 @@ +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details +version: 2 + +sphinx: + configuration: docs/conf.py + +python: + version: 3.7 + install: + - requirements: docs/requirements.txt diff --git a/.travis.yml b/.travis.yml index 987f5e1..69b90ef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,11 @@ matrix: include: - python: 3.7 +before_install: + - pip install poetry + install: - - pip install -e . - - pip install -r requirements.txt + - poetry install script: - - make + - poetry run make diff --git a/LICENSE b/LICENSE index 5759baa..f8a5884 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,7 @@ The MIT License (MIT) Copyright (c) 2018 Hyperion Gray +Copyright (c) 2022 Heraldo Lucena Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile index 8d7b78a..dde37e5 100644 --- a/Makefile +++ b/Makefile @@ -1,24 +1,27 @@ -all: mypy-generate test-generate generate test-import mypy-cdp test-cdp +# The targets in this makefile should be executed inside Poetry, i.e. `poetry run make +# docs`. + +.PHONY: docs + +default: mypy-generate test-generate generate test-import mypy-cdp test-cdp + +docs: + $(MAKE) -C docs html generate: - python generator/generate.py + python cdpgen/generate.py mypy-cdp: mypy cdp/ mypy-generate: - mypy generator/ - -publish: - rm -fr dist chrome_devtools_protocol.egg-info - $(PYTHON) setup.py sdist - twine upload dist/* + mypy cdpgen/ test-cdp: pytest test/ test-generate: - pytest generator/ + pytest cdpgen/ test-import: python -c 'import cdp; print(cdp.accessibility)' diff --git a/README.md b/README.md index ebdbe49..8343608 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,160 @@ -# PyCDP +# Python CDP +#### Currently supports CDP [r1429850][2]. -[![PyPI](https://img.shields.io/pypi/v/chrome-devtools-protocol.svg)](https://pypi.org/project/chrome-devtools-protocol/) -![Python Versions](https://img.shields.io/pypi/pyversions/chrome-devtools-protocol) -![MIT License](https://img.shields.io/github/license/HyperionGray/python-chrome-devtools-protocol.svg) -[![Build Status](https://img.shields.io/travis/com/HyperionGray/python-chrome-devtools-protocol.svg?branch=master)](https://travis-ci.com/HyperionGray/python-chrome-devtools-protocol) -[![Read the Docs](https://img.shields.io/readthedocs/py-cdp.svg)](https://py-cdp.readthedocs.io) - -Python Chrome DevTools Protocol (shortened to PyCDP) is a library that provides +Python CDP Generator (shortened to PyCDP) is a library that provides Python wrappers for the types, commands, and events specified in the [Chrome -DevTools Protocol](https://github.com/ChromeDevTools/devtools-protocol/). +DevTools Protocol][1]. The Chrome DevTools Protocol provides for remote control of a web browser by sending JSON messages over a WebSocket. That JSON format is described by a machine-readable specification. This specification is used to automatically generate the classes and methods found in this library. -You could write a CDP client by connecting a WebSocket and then sending JSON -objects, but this would be tedious and error-prone: the Python interpreter would -not catch any typos in your JSON objects, and you wouldn't get autocomplete for -any parts of the JSON data structure. By providing a set of native Python -wrappers, this project makes it easier and faster to write CDP client code. +## Installation +You can install this library as a dependency on your project with: +``` +pip install git+https://github.com/HMaker/python-cdp.git@latest +``` +Change the git tag `@latest` if you need another version. To install for development, clone this +repository, install [Poetry][5] package manager and run `poetry install` to install dependencies. + +## Usage +If all you want is automate Chrome right now, PyCDP includes a low-level client for asyncio and twisted: +```python +import asyncio +from pycdp import cdp +from pycdp.browser import ChromeLauncher +from pycdp.asyncio import connect_cdp + +async def main(): + chrome = ChromeLauncher( + binary='/usr/bin/google-chrome', # linux path + args=['--remote-debugging-port=9222', '--incognito'] + ) + # ChromeLauncher.launch() is blocking, run it on a background thread + await asyncio.get_running_loop().run_in_executor(None, chrome.launch) + conn = await connect_cdp('http://localhost:9222') + target_id = await conn.execute(cdp.target.create_target('about:blank')) + target_session = await conn.connect_session(target_id) + await target_session.execute(cdp.page.enable()) + # you may use "async for target_session.listen()" to listen multiple events, here we listen just a single event. + with target_session.safe_wait_for(cdp.page.DomContentEventFired) as navigation: + await target_session.execute(cdp.page.navigate('https://chromedevtools.github.io/devtools-protocol/')) + await navigation + dom = await target_session.execute(cdp.dom.get_document()) + node = await target_session.execute(cdp.dom.query_selector(dom.node_id, 'p')) + js_node = await target_session.execute(cdp.dom.resolve_node(node)) + print((await target_session.execute(cdp.runtime.call_function_on('function() {return this.innerText;}', js_node.object_id, return_by_value=True)))[0].value) + await target_session.execute(cdp.page.close()) + await conn.close() + await asyncio.get_running_loop().run_in_executor(None, chrome.kill) + +asyncio.run(main()) +``` +the twisted client requires [twisted][6] and [autobahn][7] packages: +```python +from twisted.python.log import err +from twisted.internet import reactor, defer, threads +from pycdp import cdp +from pycdp.browser import ChromeLauncher +from pycdp.twisted import connect_cdp + +async def main(): + chrome = ChromeLauncher( + binary='C:\Program Files\Google\Chrome\Application\chrome.exe', # windows path + args=['--remote-debugging-port=9222', '--incognito'] + ) + await threads.deferToThread(chrome.launch) + conn = await connect_cdp('http://localhost:9222', reactor) + target_id = await conn.execute(cdp.target.create_target('about:blank')) + target_session = await conn.connect_session(target_id) + await target_session.execute(cdp.page.enable()) + await target_session.execute(cdp.page.navigate('https://chromedevtools.github.io/devtools-protocol/')) + async with target_session.wait_for(cdp.page.DomContentEventFired): + dom = await target_session.execute(cdp.dom.get_document()) + node = await target_session.execute(cdp.dom.query_selector(dom.node_id, 'p')) + js_node = await target_session.execute(cdp.dom.resolve_node(node)) + print((await target_session.execute(cdp.runtime.call_function_on('function() {return this.innerText;}', js_node.object_id, return_by_value=True)))[0].value) + await target_session.execute(cdp.page.close()) + await conn.close() + await threads.deferToThread(chrome.kill) + +def main_error(failure): + err(failure) + reactor.stop() + +d = defer.ensureDeferred(main()) +d.addErrback(main_error) +d.addCallback(lambda *args: reactor.stop()) +reactor.run() +``` + +You also can use just the built-in CDP type wrappers with `import pycdp.cdp` on your own client implementation. If you want to try a different CDP version you can build new type wrappers with `cdpgen` command: +``` +usage: cdpgen + +Generate Python types for the Chrome Devtools Protocol (CDP) specification. + +optional arguments: + -h, --help show this help message and exit + --browser-protocol BROWSER_PROTOCOL + JSON file for the browser protocol + --js-protocol JS_PROTOCOL + JSON file for the javascript protocol + --output OUTPUT output path for the generated Python modules + +JSON files for the CDP spec can be found at https://github.com/ChromeDevTools/devtools-protocol/tree/master/json +``` +Example: +```sh +cdpgen --browser-protocol browser_protocol.json --js-protocol js_protocol.json --output /tmp/cdp +``` +You can then include the `/tmp/cdp` package in your project and import it like the builtin CDP types. + +### Updating built-in CDP wrappers +The `update-cdp.sh` script generates the builtin CDP wrappers, the `pycdp.cdp` package, by automatically fetching CDP protocol specifications from the [ChromeDevTools][8] repostitory. + +**To generate types for the latest version:** +```shell +./update-cdp.sh +``` +**To generate types for a specific version, you must provide full commit hash:** +```shell +./update-cdp.sh 4dd6c67776f43f75bc9b19f09618c151621c6ed9 +``` +P.S. Don't forget to make it executable by running `chmod +x update-cdp.sh` + +## Implementation of a CDP client +The `pycdp.cdp` package follows same structure of CDP domains, each domain is a Python module and each command a function in that module. + +Each function is a generator with a single yield which is a Python dict, on the CDP wire format, +containing the message that should be sent to the browser, on resumption the generator receives the message from browser: +```python +import cdp -**This library does not perform any I/O!** In order to maximize -flexibility, this library does not actually handle any network I/O, such as -opening a socket or negotiating a WebSocket protocol. Instead, that -responsibility is left to higher-level libraries, for example -[trio-chrome-devtools-protocol](https://github.com/hyperiongray/trio-chrome-devtools-protocol). +# Get all CDP targets +command = cdp.target.get_targets() # this is a generator +raw_cdp_request = next(command) # receive the yield +raw_cdp_response = send_cdp_request(raw_cdp_request) # you implement send_cdp_request, raw_cdp_request is the JSON object that should be sent to browser +try: + command.send(raw_cdp_response) # send the response to the generator where raw_cdp_response is the JSON object received from browser, it will raise StopIteration + raise RuntimeError("the generator didnt exit!") # this shouldnt happen +except StopIteration as result: + response = result.value # the parsed response to Target.get_targets() command +print(response) +``` +For implementation details check out the [docs][3]. -For more information, see the [complete documentation](https://py-cdp.readthedocs.io). +
+
+PyCDP is licensed under the MIT License. +
-define hyperion gray +[1]: https://chromedevtools.github.io/devtools-protocol/ +[2]: https://github.com/ChromeDevTools/devtools-protocol/tree/e1bdcc8cda9709838002a1516058ec8b266cbe88 +[3]: docs/getting_started.rst +[4]: https://github.com/hyperiongray/trio-chrome-devtools-protocol +[5]: https://python-poetry.org/docs/ +[6]: https://pypi.org/project/Twisted/ +[7]: https://pypi.org/project/autobahn/ +[8]: https://github.com/ChromeDevTools/devtools-protocol diff --git a/cdp/__init__.py b/cdp/__init__.py deleted file mode 100644 index ec58a08..0000000 --- a/cdp/__init__.py +++ /dev/null @@ -1,53 +0,0 @@ -''' -DO NOT EDIT THIS FILE - -This file is generated from the CDP specification. If you need to make changes, -edit the generator and regenerate all of the modules. -''' - -import cdp.util - -import cdp.accessibility -import cdp.animation -import cdp.application_cache -import cdp.audits -import cdp.background_service -import cdp.browser -import cdp.css -import cdp.cache_storage -import cdp.cast -import cdp.console -import cdp.dom -import cdp.dom_debugger -import cdp.dom_snapshot -import cdp.dom_storage -import cdp.database -import cdp.debugger -import cdp.device_orientation -import cdp.emulation -import cdp.fetch -import cdp.headless_experimental -import cdp.heap_profiler -import cdp.io -import cdp.indexed_db -import cdp.input -import cdp.inspector -import cdp.layer_tree -import cdp.log -import cdp.memory -import cdp.network -import cdp.overlay -import cdp.page -import cdp.performance -import cdp.profiler -import cdp.runtime -import cdp.schema -import cdp.security -import cdp.service_worker -import cdp.storage -import cdp.system_info -import cdp.target -import cdp.tethering -import cdp.tracing -import cdp.web_audio -import cdp.web_authn diff --git a/cdp/accessibility.py b/cdp/accessibility.py deleted file mode 100644 index 8c38e79..0000000 --- a/cdp/accessibility.py +++ /dev/null @@ -1,460 +0,0 @@ -''' -DO NOT EDIT THIS FILE - -This file is generated from the CDP specification. If you need to make changes, -edit the generator and regenerate all of the modules. - -Domain: Accessibility -Experimental: True -''' - -from cdp.util import event_class, T_JSON_DICT -from dataclasses import dataclass -import enum -import typing - -from . import dom -from . import runtime - - -class AXNodeId(str): - ''' - Unique accessibility node identifier. - ''' - def to_json(self) -> str: - return self - - @classmethod - def from_json(cls, json: str) -> 'AXNodeId': - return cls(json) - - def __repr__(self): - return 'AXNodeId({})'.format(super().__repr__()) - - -class AXValueType(enum.Enum): - ''' - Enum of possible property types. - ''' - BOOLEAN = "boolean" - TRISTATE = "tristate" - BOOLEAN_OR_UNDEFINED = "booleanOrUndefined" - IDREF = "idref" - IDREF_LIST = "idrefList" - INTEGER = "integer" - NODE = "node" - NODE_LIST = "nodeList" - NUMBER = "number" - STRING = "string" - COMPUTED_STRING = "computedString" - TOKEN = "token" - TOKEN_LIST = "tokenList" - DOM_RELATION = "domRelation" - ROLE = "role" - INTERNAL_ROLE = "internalRole" - VALUE_UNDEFINED = "valueUndefined" - - def to_json(self) -> str: - return self.value - - @classmethod - def from_json(cls, json: str) -> 'AXValueType': - return cls(json) - - -class AXValueSourceType(enum.Enum): - ''' - Enum of possible property sources. - ''' - ATTRIBUTE = "attribute" - IMPLICIT = "implicit" - STYLE = "style" - CONTENTS = "contents" - PLACEHOLDER = "placeholder" - RELATED_ELEMENT = "relatedElement" - - def to_json(self) -> str: - return self.value - - @classmethod - def from_json(cls, json: str) -> 'AXValueSourceType': - return cls(json) - - -class AXValueNativeSourceType(enum.Enum): - ''' - Enum of possible native property sources (as a subtype of a particular AXValueSourceType). - ''' - FIGCAPTION = "figcaption" - LABEL = "label" - LABELFOR = "labelfor" - LABELWRAPPED = "labelwrapped" - LEGEND = "legend" - TABLECAPTION = "tablecaption" - TITLE = "title" - OTHER = "other" - - def to_json(self) -> str: - return self.value - - @classmethod - def from_json(cls, json: str) -> 'AXValueNativeSourceType': - return cls(json) - - -@dataclass -class AXValueSource: - ''' - A single source for a computed AX property. - ''' - #: What type of source this is. - type: 'AXValueSourceType' - - #: The value of this property source. - value: typing.Optional['AXValue'] = None - - #: The name of the relevant attribute, if any. - attribute: typing.Optional[str] = None - - #: The value of the relevant attribute, if any. - attribute_value: typing.Optional['AXValue'] = None - - #: Whether this source is superseded by a higher priority source. - superseded: typing.Optional[bool] = None - - #: The native markup source for this value, e.g. a