Benedict Thekkel
fastai 2.7.15
fastAIcourse 0.0.170 /home/ben/BENEDICT_Only/DL/FastAI_course
fastbook 0.0.29
fastcore 1.5.45
fastdownload 0.0.7
fastjsonschema 2.19.1
fastprogress 1.0.3from fastcore.test import *
from rich.console import Console as _Console
def _render(renderable, width=120):
"Render to plain text at a fixed width, so wrapping cannot vary the result"
c = _Console(width=width, force_terminal=False, no_color=True)
with c.capture() as cap: c.print(renderable)
return cap.get()
# `git status -s` pads the code to two columns, so an unstaged modification opens with
# a space. Stripping the output eats that space on the FIRST line only, shifting that
# one path by a character. This fixture leads with exactly that case.
PORCELAIN = (" M nbdevAuto/_modidx.py\n"
"A nbdevAuto/new.py\n"
" D gone.py\n"
"R old.py -> new.py\n"
"?? scratch.txt\n"
"UU conflict.py\n")
table, n = _status_table(PORCELAIN)
test_eq(n, 6)
out = _render(table)
for path in ["nbdevAuto/_modidx.py", "nbdevAuto/new.py", "gone.py",
"old.py -> new.py", "scratch.txt", "conflict.py"]:
assert path in out, f"path mangled, {path!r} not in:\n{out}"
for word in ["modified", "added", "deleted", "renamed", "untracked", "conflict"]:
assert word in out, f"missing label: {word}"# Paths and docstrings are data, not markup: rich reads a bracketed word as a style
# tag and swallows it. "[gh]" is the case that actually bit, in the `h` footer.
table, _ = _status_table("?? weird[gh]name.ipynb\n")
assert "weird[gh]name.ipynb" in _render(table)
assert "nbdevAuto[gh]" in _render(escape("pip install 'nbdevAuto[gh]'"))
# Durations change scale rather than printing 0.02s or 125.0s
_now = perf_counter()
assert _dur(_now).endswith("ms")
assert _dur(_now - 5).endswith("s") and not _dur(_now - 5).endswith("ms")
assert _dur(_now - 125).startswith("2m")# The summarisers turn tool chatter into one SHORT line each: the whole point is that a
# stage occupies one row of the terminal, so anything long enough to wrap defeats it.
test_eq(_version_summary("Old version: 0.0.185\nNew version: 0.0.186\n"), ["0.0.185 -> 0.0.186"])
_TESTOUT = "Success.\n" + "".join(f"{i:02d}_nb.ipynb: 0 secs\n" for i in range(26))
test_eq(_test_summary(_TESTOUT), ["26 notebooks, all under 1s"])
test_eq(_test_summary(_TESTOUT + "slow.ipynb: 12.5 secs\n"), ["27 notebooks, slowest slow.ipynb 12.5s"])
test_eq(_test_summary("Success.\n"), ["Success."])
_PUSH = ("Enumerating objects: 13, done.\n"
"Compressing objects: 100% (6/6), done.\n"
"Writing objects: 100% (7/7), 5.29 KiB | 2.65 MiB/s, done.\n"
"To github.com:bthek1/bthek1_blog.git\n"
" 797ca71..76bea07 main -> main\n")
test_eq(_push_summary(_PUSH), ["797ca71..76bea07 main -> main", "5.29 KiB"])
test_eq(_push_summary("Everything up-to-date\n"), ["everything up-to-date"])
test_eq(_commit_summary("[main 76bea07] update x\n 3 files changed, 30 insertions(+), 18 deletions(-)\n"),
["main 76bea07 3 files +30 -18"])
test_eq(_commit_summary("[main abc1234] x\n 1 file changed, 2 insertions(+)\n"),
["main abc1234 1 file +2 -0"])
# Every summariser has to fit the detail column, or the one-line grid wraps.
for _s in (_test_summary(_TESTOUT) + _push_summary(_PUSH) + _version_summary("Old version: 0.0.1\nNew version: 0.0.2\n")):
assert len(_s) <= _W_DETAIL, f"{_s!r} is {len(_s)} chars, over {_W_DETAIL}"
# Staged paths are trimmed to fit rather than wrapped, and the remainder is counted.
test_eq(_staged_summary(" M a.py\n M b.py\n"), ["2 staged: a.py, b.py"])
_many = _staged_summary("".join(f" M some/directory/file{i}.py\n" for i in range(9)))[0]
assert _many.startswith("9 staged:") and _many.endswith(("+7", "+8")), _many
assert len(_many) <= _W_DETAIL, _many
test_eq(_staged_summary(""), [])
# Without -m the subject used to be the porcelain blob itself. Mixed codes fall back to "update".
test_eq(_auto_msg(" M career/pipeline.md\n M career/companies.md\n"),
"update career/pipeline.md, career/companies.md")
test_eq(_auto_msg("A a.py\nA b.py\nA c.py\nA d.py\n"), "add a.py, b.py, c.py and 1 more")
test_eq(_auto_msg("R old.py -> new.py\n"), "rename new.py")
test_eq(_auto_msg(" M a.py\nA b.py\n"), "update a.py, b.py")
test_eq(_auto_msg(""), "update")
# A non-zero exit has to reach `_step`, or the green OK is a lie. Note `git rev-parse
# --anything` exits 0 INSIDE a repo, so an unknown subcommand is the portable failure.
try:
_run("git", "not-a-real-subcommand")
assert False, "a failing command must raise"
except StepFailed as e:
assert e.code != 0 and e.output.strip()
# `_quiet` must swallow SUBPROCESS output too. redirect_stdout alone does not: a child
# writing to fd 1 goes straight past it, which is how pandoc used to leak 16 lines into
# the middle of a run.
with _quiet() as _buf:
print("python-level")
subprocess.run([sys.executable, "-c", "print('subprocess-level')"], check=True)
_got = _buf.getvalue()
assert "python-level" in _got and "subprocess-level" in _got, _got
# One stage, one line: label, outcome, duration and detail all on the same row.
with _step(1, 2, "demo") as d: d += ["one thing"]
_skip(2, 2, "demo skipped", "nothing to do")
try:
with _step(2, 2, "demo fail"): raise StepFailed("git push", 128, "fatal: no upstream")
except StepFailed: pass
# The grid has to hold inside 88 columns. Rendered for real and measured, because the
# widths are the whole reason the one-line form is readable.
# A console pinned to the target width, so the assertion measures the design and not
# whatever width the test runner happens to have.
_saved_console, console = console, Console(width=_WIDTH, no_color=True)
with console.capture() as _cap:
with _step(1, 6, "bump version (part 2)") as d: d += _version_summary("Old version: 0.3.67\nNew version: 0.3.68\n")
with _step(3, 6, "nbdev_test") as d: d += _test_summary(_TESTOUT + "slow.ipynb: 12.5 secs\n")
with _step(1, 3, "git add") as d: d += _staged_summary(" M README.md\n M nbdevAuto/__init__.py\n M nbs/index.ipynb\n")
with _step(3, 3, "git push") as d: d += _push_summary(_PUSH)
_skip(2, 3, "git commit", "nothing staged")
console = _saved_console
_rendered = [re.sub(r"\x1b\[[0-9;]*m", "", l) for l in _cap.get().splitlines() if l.strip()]
test_eq(len(_rendered), 5) # five stages, five lines
for _l in _rendered: assert len(_l) <= _WIDTH, f"{len(_l)} cols: {_l!r}"
# and the outcome really is on the same row as the label
assert all(any(w in l for w in ("OK", "SKIP")) for l in _rendered), _rendered*Bump version part p, then export, test and clean the notebooks, refreshing _quarto.yml and README*
@call_parse
def prep(
p:int = 2, # Increment Part
):
"Bump version part `p`, then export, test and clean the notebooks, refreshing _quarto.yml and README"
import nbdev.test, nbdev.clean, nbdev.quarto, nbdev.release
_banner("prep")
t0 = perf_counter()
with _step(1, 6, f"bump version (part {p})") as d, _quiet() as out:
nbdev.release.nbdev_bump_version(p)
d += _version_summary(out.getvalue())
with _step(2, 6, "nbdev_export"), _quiet():
nbdev.quarto.nbdev_export.__wrapped__()
with _step(3, 6, "nbdev_test") as d, _quiet() as out:
nbdev.test.nbdev_test.__wrapped__(
n_workers = 8, # Number of workers
timing = True, # Time each notebook to see which are slow
)
d += _test_summary(out.getvalue())
with _step(4, 6, "nbdev_clean"), _quiet():
nbdev.clean.nbdev_clean.__wrapped__()
with _step(5, 6, "refresh_quarto_yml"), _quiet():
nbdev.quarto.refresh_quarto_yml()
with _step(6, 6, "nbdev_readme"), _quiet():
nbdev.quarto.nbdev_readme.__wrapped__(chk_time=True)
_done("prep", t0)git add, commit and push. Without -m the message is built from the staged paths
@call_parse
def gacp(
m:str = '', # Commit message
):
"git add, commit and push. Without `-m` the message is built from the staged paths"
_banner("gacp")
t0 = perf_counter()
# NOT via `_run`: `git status -s` pads the code to two columns, and the capture has
# to keep that leading space or the first path shifts by one.
with _step(1, 3, "git add") as d:
_run("git", "add", ".")
status = subprocess.check_output(["git", "status", "-s"]).decode('utf-8')
d += _staged_summary(status)
staged = bool(status.strip())
if staged:
with _step(2, 3, "git commit") as d:
d += _commit_summary(_run("git", "commit", "-m", m if m != '' else _auto_msg(status)))
else:
# `git commit` exits 1 with nothing staged, which is now a hard failure rather
# than a silent one. There may still be unpushed commits, so go on to the push.
_skip(2, 3, "git commit", "nothing staged")
with _step(3, 3, "git push") as d:
d += _push_summary(_run("git", "push"))
_done("gacp", t0)def status():
"Show the working tree state"
import subprocess
def _git(*a, **kw): return subprocess.check_output(["git", *a], **kw).decode('utf-8').strip()
_banner("status")
branch = _git("branch", "--show-current") or "DETACHED"
line = f"on [bold]{branch}[/bold]"
try:
behind, ahead = _git("rev-list", "--left-right", "--count", "@{upstream}...HEAD",
stderr=subprocess.DEVNULL).split()
if behind != "0": line += f" [yellow]behind {behind}[/yellow]"
if ahead != "0": line += f" [cyan]ahead {ahead}[/cyan]"
if behind == ahead == "0": line += " [dim]in step with upstream[/dim]"
except subprocess.CalledProcessError:
line += " [dim]no upstream[/dim]"
console.print(line)
# NOT via _git: `git status -s` pads the code to two columns, so stripping the
# output would eat the leading space of the first line and shift its path by one.
porcelain = subprocess.check_output(["git", "status", "-s"]).decode('utf-8')
if not porcelain:
console.print("[green]clean[/green]")
return
table, n = _status_table(porcelain)
console.print(table)
console.print(f"[dim]{n} path{'' if n == 1 else 's'} changed[/dim]")prep then gacp: the everyday command for shipping a notebook change
def release_git():
"Bump the MINOR version, then tag and create a GitHub release"
import nbdev.release
_banner("gitrelease")
t0 = perf_counter()
with _step(1, 2, "bump minor version") as d, _quiet() as out:
nbdev.release.nbdev_bump_version(1)
d += _version_summary(out.getvalue())
with _step(2, 2, "tag and create the GitHub release"):
nbdev.release.release_git()
_done("gitrelease", t0)Build the sdist and wheel, then upload to PyPI with twine. CI publishes on push, so this is the manual path
def release_pypi():
"Build the sdist and wheel, then upload to PyPI with twine. CI publishes on push, so this is the manual path"
import nbdev.release
_banner("piprelease")
t0 = perf_counter()
# Deliberately not quieted: a twine upload is worth watching line by line.
with _step(1, 1, "build and upload to PyPI"):
nbdev.release.release_pypi()
_done("piprelease", t0)Print every console script this package installs, with its help
def help_output():
"Print every console script this package installs, with its help"
from importlib.metadata import distribution
try:
eps = [e for e in distribution("nbdevAuto").entry_points if e.group == "console_scripts"]
except Exception:
# Running from a source tree with nothing installed: fall back to fastcore.
from fastcore.xtras import console_help
return console_help('nbdevAuto')
_banner("commands")
t = Table(box=box.SIMPLE, show_header=False, pad_edge=False, expand=False)
t.add_column("command", style="bold cyan", no_wrap=True)
t.add_column("does", overflow="fold")
for e in sorted(eps, key=lambda e: e.name):
try: doc = escape((e.load().__doc__ or "").strip().splitlines()[0])
except Exception: doc = "[red]could not load[/red]"
t.add_row(e.name, doc)
console.print(t)
console.print(escape("ghstatus needs the gh extra: pip install 'nbdevAuto[gh]'"), style="dim")requirements = fastdownload>=0.0.5,<2 fastcore>=1.5.29,<1.6 torchvision matplotlib pandas requests pyyaml fastprogress>=0.2.4 pillow>6.0.0 scikit-learn scipy spacy<4 packaging fastbook twine conda_requirements = pytorch>=1.7,<2.1 dev_requirements = ipywidgets pytorch-lightning pytorch-ignite transformers sentencepiece tensorboard pydicom catalyst flask_compress captum>=0.3 flask wandb kornia scikit-image neptune-client comet_ml albumentations opencv-python pyarrow catalyst ninja timm>=0.6.2.dev accelerate>=0.10.0