C:\Users\User\Desktop\sciCli\amiclimate>pytest
=================================================================== test session starts ===================================================================
platform win32 -- Python 3.12.3, pytest-8.2.1, pluggy-1.5.0
rootdir: C:\Users\User\Desktop\sciCli\amiclimate
collected 74 items
test\test_un.py ....s
DevTools listening on ws://127.0.0.1:52700/devtools/browser/530bc3ab-8e9e-47b2-934a-c3abeef9a0bd
DevTools listening on ws://127.0.0.1:52728/devtools/browser/07863b7e-36ea-4d91-8133-b214d06c622d
DevTools listening on ws://127.0.0.1:52751/devtools/browser/e4f7c27c-fe42-4f4c-93ae-c287db50397b
DevTools listening on ws://127.0.0.1:52772/devtools/browser/4a8f4f78-fc19-4dfb-ab65-47764787edf9
......sss..
DevTools listening on ws://127.0.0.1:52801/devtools/browser/b82f4e76-d6da-4903-8605-46cfc4a40838
DevTools listening on ws://127.0.0.1:52820/devtools/browser/430b7e76-e0ae-4bd4-91b7-a9e5fb7e9316
DevTools listening on ws://127.0.0.1:52841/devtools/browser/95baafb9-9c1b-4d7f-870f-6e813f77065a
DevTools listening on ws://127.0.0.1:52864/devtools/browser/88888071-a617-44eb-880a-515913ab09bb
DevTools listening on ws://127.0.0.1:52887/devtools/browser/9dab2476-813a-4b73-bd9e-81ddf73e4596
DevTools listening on ws://127.0.0.1:52906/devtools/browser/3535dee2-c75f-4d2b-95d4-ea2539528aea
DevTools listening on ws://127.0.0.1:52926/devtools/browser/f95a31f7-44f5-43da-bb97-2fe7dc037a0c
DevTools listening on ws://127.0.0.1:52945/devtools/browser/c4982276-5204-403a-8bdb-6dc2f51fd652
DevTools listening on ws://127.0.0.1:52965/devtools/browser/3aa7d266-93c9-4aea-83cb-531375108343
.
DevTools listening on ws://127.0.0.1:52984/devtools/browser/16420088-7498-41f8-a7a1-9ad293e7e586
F............FF................FF..ss.s....s.Fsssssssss.. [100%]
======================================================================== FAILURES =========================================================================
______________________________________________ TestIPCC.test_download_wg_chapter_spm_ts_using_dict_IMPORTANT ______________________________________________
self = <test.test_un.TestIPCC testMethod=test_download_wg_chapter_spm_ts_using_dict_IMPORTANT>
def test_download_wg_chapter_spm_ts_using_dict_IMPORTANT(self):
"""downlaods all parts of WG reports
writes:
gatsby_raw.html
gatsby_raw.html
de_gatsby.html
para_list
"""
reports = [
IP_WG1,
# IP_WG2,
# IP_WG3,
]
chapters = [
# SPM,
# TS,
"chapter-1",
# "chapter-2",
# "chapter-3",
# "chapter-4",
# "chapter-5",
# "chapter-6",
# "chapter-7",
# "chapter-8",
# "chapter-9",
# "chapter-10",
# "chapter-11",
# "chapter-12",
# "chapter-13",
# "chapter-14",
# "chapter-15",
# "chapter-16",
# "chapter-17",
# "chapter-18",
# "chapter-19",
]
# ipcc_dict = IPCC_DICT.get_ipcc_dict()
# ar6_url = ipcc_dict.get()
web_publisher = IPCCGatsby()
for report in reports:
wg_url = f"{AR6_URL}{report}/"
print(f"report: {report}")
for chap in chapters:
print(f"chapter: {chap}")
outdir = Path(TEMP_DIR, report, chap)
IPCC.download_save_chapter(report, chap, wg_url, outdir=TEMP_DIR, sleep=1)
raw_outfile = Path(outdir, f"{GATSBY_RAW}.html")
FileLib.assert_exist_size(raw_outfile, minsize=20000, abort=False)
gatsby_file = Path(outdir, f"{GATSBY_RAW}.html")
html_elem = web_publisher.remove_unnecessary_markup(gatsby_file)
assert html_elem is not None, f"{gatsby_file} should not give None html"
body = HtmlLib.get_body(html_elem)
> elems = body.xpath(".//*")
E AttributeError: 'NoneType' object has no attribute 'xpath'
test\test_un.py:651: AttributeError
------------------------------------------------------------------ Captured stdout call -------------------------------------------------------------------
report: wg1
chapter: chapter-1
web publisher assumed to be <class 'climate.ipcc.IPCCGatsby'>
Fetching page source from URL: https://www.ipcc.ch/report/ar6/wg1/chapter/chapter-1
no xpath_list specified
no output html
elements in lxml_root: 40
writing C:\Users\User\Desktop\sciCli\amiclimate\temp\wg1\chapter-1\gatsby_raw.html
Quitting the driver...
DONE
//div[contains(@class, 'col-12')] removes 0 elems
//div[@data-gatsby-image-wrapper]/div[@aria-hidden='true'] removes 0 elems
______________________________________________________________ TestIPCC.test_ipcc_syr_lr_toc ______________________________________________________________
self = <test.test_un.TestIPCC testMethod=test_ipcc_syr_lr_toc>
def test_ipcc_syr_lr_toc(self):
"""analyses contents for IPCC syr longer report
"""
"""
<!-- TOC (from UNFCCC)-->
<div class="toc">
<div>
<span>Decision</span><span>Page</span></a>
</div>
<nav role="doc-toc">
<ul>
<li>
<a href="../Decision_1_CMA_3/split.html"><span class="descres-code">1/CMA.3</span><span
class="descres-title">Glasgow Climate Pact</span></a>
</li>
...
</ul>
</nav>
</div>
"""
report = 'longer-report'
syr_lr_content = Path(Resources.TEST_RESOURCES_DIR, IPCC_DIR, CLEANED_CONTENT, SYR,
SYR_LR, HTML_WITH_IDS_HTML)
assert syr_lr_content.exists()
lr_html = ET.parse(str(syr_lr_content), HTMLParser())
assert lr_html is not None
body = HtmlLib.get_body(lr_html)
header_h1 = body.xpath("div//h1")[0]
assert header_h1 is not None
header_h1_text = header_h1.text
toc_title = "SYR Longer Report"
> assert header_h1_text == toc_title
E AssertionError: assert 'Chapter 3: M...ng-term goals' == 'SYR Longer Report'
E
E - SYR Longer Report
E + Chapter 3: Mitigation pathways compatible with long-term goals
C:\Users\User\Desktop\sciCli\amiclimate\test\test_un.py:1224: AssertionError
___________________________________________________________ TestIPCC.test_ipcc_syr_lr_toc_full ____________________________________________________________
self = <test.test_un.TestIPCC testMethod=test_ipcc_syr_lr_toc_full>
def test_ipcc_syr_lr_toc_full(self):
"""creates toc recursively for IPCC syr longer report
"""
filename = HTML_WITH_IDS_HTML
syr_lr_content = Path(Resources.TEST_RESOURCES_DIR, IPCC_DIR, CLEANED_CONTENT, SYR,
SYR_LR, filename)
lr_html = ET.parse(str(syr_lr_content), HTMLParser())
body = HtmlLib.get_body(lr_html)
publisher = IPCCGatsby()
toc_html, ul = publisher.make_header_and_nav_ul(body)
level = 0
> publisher.analyse_containers(body, level, ul, filename=filename)
C:\Users\User\Desktop\sciCli\amiclimate\test\test_un.py:1262:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
C:\Users\User\Desktop\sciCli\amiclimate\climate\ipcc.py:1347: in analyse_containers
self.add_container_infp_to_tree(debug, filename, h_container, level, texts, ul)
C:\Users\User\Desktop\sciCli\amiclimate\climate\ipcc.py:1365: in add_container_infp_to_tree
self.analyse_containers(h_container, level + 1, ul1, filename=filename)
C:\Users\User\Desktop\sciCli\amiclimate\climate\ipcc.py:1347: in analyse_containers
self.add_container_infp_to_tree(debug, filename, h_container, level, texts, ul)
C:\Users\User\Desktop\sciCli\amiclimate\climate\ipcc.py:1365: in add_container_infp_to_tree
self.analyse_containers(h_container, level + 1, ul1, filename=filename)
C:\Users\User\Desktop\sciCli\amiclimate\climate\ipcc.py:1347: in analyse_containers
self.add_container_infp_to_tree(debug, filename, h_container, level, texts, ul)
C:\Users\User\Desktop\sciCli\amiclimate\climate\ipcc.py:1365: in add_container_infp_to_tree
self.analyse_containers(h_container, level + 1, ul1, filename=filename)
C:\Users\User\Desktop\sciCli\amiclimate\climate\ipcc.py:1347: in analyse_containers
self.add_container_infp_to_tree(debug, filename, h_container, level, texts, ul)
C:\Users\User\Desktop\sciCli\amiclimate\climate\ipcc.py:1365: in add_container_infp_to_tree
self.analyse_containers(h_container, level + 1, ul1, filename=filename)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <climate.ipcc.IPCCGatsby object at 0x0000018EC5678BF0>, container = <Element div at 0x18ece457110>, level = 4, ul = <Element ul at 0x18ef1aaa3c0>
filename = 'html_with_ids.html', debug = False
def analyse_containers(self, container, level, ul, filename=None, debug=False):
"""Part of ToC making"""
> container_xpath = f".//div[contains(@class,'{self.container_levels[level]}')]"
E IndexError: list index out of range
C:\Users\User\Desktop\sciCli\amiclimate\climate\ipcc.py:1342: IndexError
_______________________________________________________________ TestIPCC.test_symbol_indir ________________________________________________________________
self = <test.test_un.TestIPCC testMethod=test_symbol_indir>
def test_symbol_indir(self):
infile = "**/html_with_ids.html"
outdir = f"{Path(Resources.TEMP_DIR, 'queries')}"
output = f"{Path(outdir, 'methane_norefs2')}.html"
query = "methane"
AMIClimate().run_command(
['IPCC', '--indir', "_IPCC_REPORTS", '--input', "_HTML_IDS", '--query', "methane", '--outdir', "_QUERY_OUT",
"--output", output, '--xpath',
"_NOREFS"])
> self.check_output_tree(output, expected=[60,300], xpath=".//a[@href]")
C:\Users\User\Desktop\sciCli\amiclimate\test\test_un.py:1053:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <test.test_un.TestIPCC testMethod=test_symbol_indir>, output = 'C:\\Users\\User\\Desktop\\sciCli\\amiclimate\\temp\\queries\\methane_norefs2.html'
expected = [60, 300], xpath = './/a[@href]'
def check_output_tree(self, output, expected=None, xpath=None):
assert xpath, f"must give xpath"
assert output, f"output cannot be None"
html_tree = ET.parse(output)
assert html_tree is not None, f"html_tree is None"
if expected:
pp = len(html_tree.xpath(xpath))
if type(expected) is list and len(expected) == 2:
> assert expected[0] <= pp <= expected[1], f"found {pp} elements in {output}, expected {expected}"
E AssertionError: found 57 elements in C:\Users\User\Desktop\sciCli\amiclimate\temp\queries\methane_norefs2.html, expected [60, 300]
E assert 60 <= 57
C:\Users\User\Desktop\sciCli\amiclimate\test\test_un.py:1317: AssertionError
------------------------------------------------------------------ Captured stdout call -------------------------------------------------------------------
command: ['IPCC', '--indir', '_IPCC_REPORTS', '--input', '_HTML_IDS', '--query', 'methane', '--outdir', '_QUERY_OUT', '--output', 'C:\\Users\\User\\Desktop\\sciCli\\amiclimate\\temp\\queries\\methane_norefs2.html', '--xpath', '_NOREFS']
running parse_and_process1 in util?
arg_dict: {'version': False, 'command': 'IPCC', 'debug': False, 'input': '_HTML_IDS', 'indir': '_IPCC_REPORTS', 'output': 'C:\\Users\\User\\Desktop\\sciCli\\amiclimate\\temp\\queries\\methane_norefs2.html', 'outdir': '_QUERY_OUT', 'informat': 'PDF', 'query': 'methane', 'xpath': '_NOREFS'}
home C:\Users\User
outdir C:\Users\User\Desktop\sciCli\amiclimate\temp\queries output C:\Users\User\Desktop\sciCli\amiclimate\temp\queries\methane_norefs2.html
empty list from C:/Users/User/Desktop/sciCli/amiclimate/temp/queries/C:\Users\User\Desktop\sciCli\amiclimate\temp\queries\methane_norefs2.html
inputs: 33 > [PurePosixPath('C:/Users/User/Desktop/sciCli/amiclimate/test/resources/ipcc/cleaned_content\\sr15\\Chapter01\\html_with_ids.html'), PurePosixPath('C:/Users/User/Desktop/sciCli/amiclimate/test/resources/ipcc/cleaned_content\\sr15\\Chapter02\\html_with_ids.html'), PurePosixPath('C:/Users/User/Desktop/sciCli/amiclimate/test/resources/ipcc/cleaned_content\\sr15\\Chapter03\\html_with_ids.html')]...
debug: True
report: None
chapter: None
outdir: C:\Users\User\Desktop\sciCli\amiclimate\temp\queries
output: C:\Users\User\Desktop\sciCli\amiclimate\temp\queries\methane_norefs2.html
kwargs: {}
query: ['methane']
xpath: //p[@id and not(ancestor::*[@id='references'])]
wrote: C:\Users\User\Desktop\sciCli\amiclimate\temp\queries\methane_norefs2.html
argstr: None
-------------------------------------------------------------------- Captured log call --------------------------------------------------------------------
WARNING amilib.html_args:html_args.py:24 creating HTML Args
WARNING amilib.pdf_args:pdf_args.py:81 creating PDFArgs
WARNING root:ami_args.py:120 ********** args for parse_and_process1 {'version': False, 'command': 'IPCC', 'debug': [False], 'input': ['_HTML_IDS'], 'indir': ['_IPCC_REPORTS'], 'output': ['C:\\Users\\User\\Desktop\\sciCli\\amiclimate\\temp\\queries\\methane_norefs2.html'], 'outdir': ['_QUERY_OUT'], 'kwords': None, 'chapter': None, 'informat': 'PDF', 'operation': None, 'query': ['methane'], 'report': None, 'xpath': ['_NOREFS']}
______________________________________________________________ TestIPCC.test_symbolic_xpaths ______________________________________________________________
self = <test.test_un.TestIPCC testMethod=test_symbolic_xpaths>
def test_symbolic_xpaths(self):
infile = str(
Path(Resources.TEST_RESOURCES_DIR, 'ipcc', 'cleaned_content', 'wg1', 'Chapter02', 'html_with_ids.html'))
outdir = f"{Path(Resources.TEMP_DIR, 'queries')}"
query = "methane"
output = f"{Path(outdir, 'methane_refs1')}.html"
AMIClimate().run_command(
['IPCC', '--input', infile, '--query', query, '--output', output, '--xpath', "_REFS"])
> self.check_output_tree(output, expected=7, xpath=".//a[@href]")
C:\Users\User\Desktop\sciCli\amiclimate\test\test_un.py:1034:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <test.test_un.TestIPCC testMethod=test_symbolic_xpaths>, output = 'C:\\Users\\User\\Desktop\\sciCli\\amiclimate\\temp\\queries\\methane_refs1.html'
expected = 7, xpath = './/a[@href]'
def check_output_tree(self, output, expected=None, xpath=None):
assert xpath, f"must give xpath"
assert output, f"output cannot be None"
html_tree = ET.parse(output)
assert html_tree is not None, f"html_tree is None"
if expected:
pp = len(html_tree.xpath(xpath))
if type(expected) is list and len(expected) == 2:
assert expected[0] <= pp <= expected[1], f"found {pp} elements in {output}, expected {expected}"
else:
> assert pp == expected, f"found {pp} elements in {output}"
E AssertionError: found 10 elements in C:\Users\User\Desktop\sciCli\amiclimate\temp\queries\methane_refs1.html
E assert 10 == 7
C:\Users\User\Desktop\sciCli\amiclimate\test\test_un.py:1319: AssertionError
------------------------------------------------------------------ Captured stdout call -------------------------------------------------------------------
command: ['IPCC', '--input', 'C:\\Users\\User\\Desktop\\sciCli\\amiclimate\\test\\resources\\ipcc\\cleaned_content\\wg1\\Chapter02\\html_with_ids.html', '--query', 'methane', '--output', 'C:\\Users\\User\\Desktop\\sciCli\\amiclimate\\temp\\queries\\methane_refs1.html', '--xpath', '_REFS']
running parse_and_process1 in util?
arg_dict: {'version': False, 'command': 'IPCC', 'debug': False, 'input': 'C:\\Users\\User\\Desktop\\sciCli\\amiclimate\\test\\resources\\ipcc\\cleaned_content\\wg1\\Chapter02\\html_with_ids.html', 'output': 'C:\\Users\\User\\Desktop\\sciCli\\amiclimate\\temp\\queries\\methane_refs1.html', 'informat': 'PDF', 'query': 'methane', 'xpath': '_REFS'}
home C:\Users\User
outdir None output C:\Users\User\Desktop\sciCli\amiclimate\temp\queries\methane_refs1.html
inputs: 1 > [PurePosixPath('C:\\Users\\User\\Desktop\\sciCli\\amiclimate\\test\\resources\\ipcc\\cleaned_content\\wg1\\Chapter02\\html_with_ids.html')]...
debug: True
report: None
chapter: None
outdir: None
output: C:\Users\User\Desktop\sciCli\amiclimate\temp\queries\methane_refs1.html
kwargs: {}
query: ['methane']
xpath: //p[@id and ancestor::*[@id='references']]
wrote: C:\Users\User\Desktop\sciCli\amiclimate\temp\queries\methane_refs1.html
argstr: None
-------------------------------------------------------------------- Captured log call --------------------------------------------------------------------
WARNING amilib.html_args:html_args.py:24 creating HTML Args
WARNING amilib.pdf_args:pdf_args.py:81 creating PDFArgs
WARNING root:ami_args.py:120 ********** args for parse_and_process1 {'version': False, 'command': 'IPCC', 'debug': [False], 'input': ['C:\\Users\\User\\Desktop\\sciCli\\amiclimate\\test\\resources\\ipcc\\cleaned_content\\wg1\\Chapter02\\html_with_ids.html'], 'indir': None, 'output': ['C:\\Users\\User\\Desktop\\sciCli\\amiclimate\\temp\\queries\\methane_refs1.html'], 'outdir': None, 'kwords': None, 'chapter': None, 'informat': 'PDF', 'operation': None, 'query': ['methane'], 'report': None, 'xpath': ['_REFS']}
____________________________________________________________ TestUNFCCC.test_make_nested_divs _____________________________________________________________
self = <test.test_un.TestUNFCCC testMethod=test_make_nested_divs>
def test_make_nested_divs(self):
"""IMPORTANT not finished"""
"""initial div files are 'flat' - all divs are siblings, Use parents in markup_dict to assemble
"""
input_dir = Path(UNFCCC_DIR, "unfcccdocuments1", "CMA_3")
infile = Path(input_dir, "1_4_CMA_3_section", f"normalized.sections.html")
> assert str(infile).endswith(
"test/resources/unfccc/unfcccdocuments1/CMA_3/1_4_CMA_3_section/normalized.sections.html")
E AssertionError: assert False
E + where False = <built-in method endswith of str object at 0x0000018E85109E70>('test/resources/unfccc/unfcccdocuments1/CMA_3/1_4_CMA_3_section/normalized.sections.html')
E + where <built-in method endswith of str object at 0x0000018E85109E70> = 'C:\\Users\\User\\Desktop\\sciCli\\amiclimate\\test\\resources\\unfccc\\unfcccdocuments1\\CMA_3\\1_4_CMA_3_section\\normalized.sections.html'.endswith
E + where 'C:\\Users\\User\\Desktop\\sciCli\\amiclimate\\test\\resources\\unfccc\\unfcccdocuments1\\CMA_3\\1_4_CMA_3_section\\normalized.sections.html' = str(WindowsPath('C:/Users/User/Desktop/sciCli/amiclimate/test/resources/unfccc/unfcccdocuments1/CMA_3/1_4_CMA_3_section/normalized.sections.html'))
C:\Users\User\Desktop\sciCli\amiclimate\test\test_un.py:1593: AssertionError
================================================================= short test summary info =================================================================
FAILED test/test_un.py::TestIPCC::test_download_wg_chapter_spm_ts_using_dict_IMPORTANT - AttributeError: 'NoneType' object has no attribute 'xpath'
FAILED test/test_un.py::TestIPCC::test_ipcc_syr_lr_toc - AssertionError: assert 'Chapter 3: M...ng-term goals' == 'SYR Longer Report'
FAILED test/test_un.py::TestIPCC::test_ipcc_syr_lr_toc_full - IndexError: list index out of range
FAILED test/test_un.py::TestIPCC::test_symbol_indir - AssertionError: found 57 elements in C:\Users\User\Desktop\sciCli\amiclimate\temp\queries\methane_norefs2.html, expected [60, 300]
FAILED test/test_un.py::TestIPCC::test_symbolic_xpaths - AssertionError: found 10 elements in C:\Users\User\Desktop\sciCli\amiclimate\temp\queries\methane_refs1.html
FAILED test/test_un.py::TestUNFCCC::test_make_nested_divs - AssertionError: assert False
================================================== 6 failed, 51 passed, 17 skipped in 828.86s (0:13:48) ===================================================