pages on `linux` available for `macos`

[vitorhcl]

I made a simple script that hits https://keith.github.io/xcode-man-pages for checking if the linux pages are available on macOS, printing the ones that return 200 and are not already present on common.

However, we can't simply move them to common, because we have to check if there are any difference between platforms.

Missing pages on section 1:

• [ ] brctl →
• [ ] chfn →
• [ ] colrm →
• [ ] compress →
• [ ] csplit →
• [ ] diff3 →
• [ ] expect →
• [ ] fuser →
• [ ] getconf →
• [ ] getopt →
• [ ] gzexe →
• [ ] ipcrm →
• [ ] lastcomm →
• [ ] libtool →
• [ ] locale →
• [ ] login →
• [ ] manpath →
• [ ] ncal →
• [ ] newgrp →
• [ ] postconf →
• [ ] postfix →
• [ ] reset →
• [ ] size →
• [ ] snmpwalk →
• [ ] strip →
• [ ] talk →
• [ ] tftp →
• [ ] ul →
• [ ] uncompress →
• [ ] unset →
• [ ] unzipsfx →
• [ ] wall →
• [ ] zforce →
• [ ] zipcloak → #12543
• [ ] zipsplit → #12544

~Missing pages on section 5:~ This section is for file formats

Missing pages on section 8:

• [ ] edquota →
• [ ] fdisk →
• [ ] halt →
• [ ] iostat →
• [ ] mknod →
• [ ] nologin →
• [ ] quotacheck →
• [ ] repquota →
• [ ] rpcinfo →
• [ ] sa →
• [ ] vipw →

The script

import argparse
import asyncio
from aiopath import AsyncPath
import aiohttp
from random import shuffle
from aioconsole import aprint


async def list_files(directory):
  files = [file async for file in AsyncPath(directory).iterdir() if file.is_file()]
  return files


async def fetch_data(file, url):
  async with aiohttp.ClientSession() as session:
      async with session.get(url) as response:
          return file, response.status


async def print_response(file, status_code, url):
  color = "\033[32m" if status_code == 200 else "\033[31m"
  reset_color = "\033[0m"
  formatted_code = f"{color}{status_code}{reset_color}"
  await aprint(
      f"File: {file.name[:-3]}; Response code: {formatted_code}; Link: {url}"
  )


async def process_file(file, url):
  file, status_code = await fetch_data(file, url)
  if status_code == 200:
      await print_response(file, status_code, url)
      return file.stem  # Return the file stem if status code is 200


async def fetch_and_print_file_data(
  linux_directory, common_directory, macos_directory, manual_section
):
  files = await list_files(linux_directory)
  present_files = [file.stem for file in await list_files(common_directory)]
  present_files.extend(file.stem for file in await list_files(macos_directory))

  await aprint(files)
  url_template = (
      "https://keith.github.io/xcode-man-pages/{}." + manual_section + ".html"
  )
  links = [
      (file, url_template.format(file.stem))
      for file in files
      if file.stem not in present_files
  ]
  shuffle(links)

  tasks = []
  for file, url in links:
      task = process_file(file, url)
      tasks.append(task)

  file_stems = await asyncio.gather(*tasks)
  todo_list = sorted(set(filter(None, file_stems)))

  await aprint(f"\nMissing pages on section {manual_section}:")
  for stem in todo_list:
      await aprint(f"- [ ] {stem} →")


async def main():
  parser = argparse.ArgumentParser()
  parser.add_argument(
      "tldr_root", type=AsyncPath, help="Root directory of TLDR pages"
  )
  parser.add_argument(
      "manual_section",
      help="Hit manual section 1, 5 or 8",
      choices=["1", "5", "8"],
  )
  args = parser.parse_args()

  pages_directory = AsyncPath(args.tldr_root) / "pages"

  await fetch_and_print_file_data(
      pages_directory / "linux",
      pages_directory / "common",
      pages_directory / "osx",
      args.manual_section,
  )


if __name__ == "__main__":
  asyncio.run(main())

[TcPirate1]

This is a good script! Only change that I'd like to point out to future users of it is that in the first function, an await keyword after the if is needed.

So instead of:

async def list_files(directory):
  files = [file async for file in AsyncPath(directory).iterdir() if file.is_file()]
  return files

It should be:

async def list_files(directory):
  files = [file async for file in AsyncPath(directory).iterdir() if await file.is_file()]
  return files

Full edited script

import argparse
import asyncio
from aiopath import AsyncPath
import aiohttp
from random import shuffle
from aioconsole import aprint


async def list_files(directory):
  files = [file async for file in AsyncPath(directory).iterdir() if await file.is_file()]
  return files


async def fetch_data(file, url):
  async with aiohttp.ClientSession() as session:
      async with session.get(url) as response:
          return file, response.status


async def print_response(file, status_code, url):
  color = "\033[32m" if status_code == 200 else "\033[31m"
  reset_color = "\033[0m"
  formatted_code = f"{color}{status_code}{reset_color}"
  await aprint(
      f"File: {file.name[:-3]}; Response code: {formatted_code}; Link: {url}"
  )


async def process_file(file, url):
  file, status_code = await fetch_data(file, url)
  if status_code == 200:
      await print_response(file, status_code, url)
      return file.stem  # Return the file stem if status code is 200


async def fetch_and_print_file_data(
  linux_directory, common_directory, macos_directory, manual_section
):
  files = await list_files(linux_directory)
  present_files = [file.stem for file in await list_files(common_directory)]
  present_files.extend(file.stem for file in await list_files(macos_directory))

  await aprint(files)
  url_template = (
      "https://keith.github.io/xcode-man-pages/{}." + manual_section + ".html"
  )
  links = [
      (file, url_template.format(file.stem))
      for file in files
      if file.stem not in present_files
  ]
  shuffle(links)

  tasks = []
  for file, url in links:
      task = process_file(file, url)
      tasks.append(task)

  file_stems = await asyncio.gather(*tasks)
  todo_list = sorted(set(filter(None, file_stems)))

  await aprint(f"\nMissing pages on section {manual_section}:")
  for stem in todo_list:
      await aprint(f"- [ ] {stem} →")


async def main():
  parser = argparse.ArgumentParser()
  parser.add_argument(
      "tldr_root", type=AsyncPath, help="Root directory of TLDR pages"
  )
  parser.add_argument(
      "manual_section",
      help="Hit manual section 1, 5 or 8",
      choices=["1", "5", "8"],
  )
  args = parser.parse_args()

  pages_directory = AsyncPath(args.tldr_root) / "pages"

  await fetch_and_print_file_data(
      pages_directory / "linux",
      pages_directory / "common",
      pages_directory / "osx",
      args.manual_section,
  )


if __name__ == "__main__":
  asyncio.run(main())

Hopefully this saves some minor headaches :)

[vitorhcl]

This is a good script! Only change that I'd like to point out to future users of it is that in the first function, an await keyword after the if is needed.

So instead of:

async def list_files(directory):
  files = [file async for file in AsyncPath(directory).iterdir() if file.is_file()]
  return files

It should be:

async def list_files(directory):
  files = [file async for file in AsyncPath(directory).iterdir() if await file.is_file()]
  return files

Hopefully this saves some minor headaches :)

Thank you for the fix! Feel free to edit my comment and modify the script :)