noaa-ocs-modeling / nemspy Goto Github PK

View Code? Open in Web Editor NEW

1.0 1.0 2.0 285 KB

NEMS configuration generation

Home Page: https://nemspy.readthedocs.io

License: Creative Commons Zero v1.0 Universal

Python 100.00%

configuration-management nems noaa nuopc python

nemspy's People

Contributors

Stargazers

Watchers

Forkers

acrosby zacharyburnett

nemspy's Issues

add parameters to `model_configure`

model_configure is currently mostly hardcoded; need to add parameters to the ModelConfiguration object:

nemspy/nemspy/configuration.py

Lines 373 to 528 in f0b9c25

 class ModelConfigurationFile(ConfigurationFile): 

 name = 'model_configure' 

 def __init__(self, start_time: datetime, duration: timedelta, sequence: RunSequence): 

 self.start_time = start_time 

 self.duration = duration 

 super().__init__(sequence) 

 def write(self, directory: PathLike, overwrite: bool = False): 

 directory = ensure_directory(directory) 

 filename = directory / self.name 

 if filename.exists(): 

 LOGGER.warning( 

 f'{"overwriting" if overwrite else "skipping"} ' f'existing file "{filename}"' 

 ) 

 if not filename.exists() or overwrite: 

 LOGGER.debug(f'writing model configuration to "{filename}"') 

 with open(filename, 'w') as output_file: 

 output_file.write(str(self)) 

 symbolic_link_filename = directory / 'atm_namelist.rc' 

 try: 

 symbolic_link_filename.symlink_to(filename) 

 except Exception as error: 

 LOGGER.warning(f'could not create symbolic link: {error}') 

 with open(symbolic_link_filename, 'w') as output_file: 

 output_file.write(str(self)) 

 def __str__(self) -> str: 

 duration_hours = round(self.duration / timedelta(hours=1)) 

 return '\n'.join( 

 [ 

 'core: gfs', 

 'print_esmf: .true.', 

 '', 

 'nhours_dfini=0', 

 '', 

 '#nam_atm +++++++++++++++++++++++++++', 

 'nlunit: 35', 

 'deltim: 900.0', 

 'fhrot: 0', 

 'namelist: atm_namelist', 

 'total_member: 1', 

 'grib_input: 0', 

 f'PE_MEMBER01: {self.sequence.processors}', 

 'PE_MEMBER02', 

 'PE_MEMBER03', 

 'PE_MEMBER04', 

 'PE_MEMBER05', 

 'PE_MEMBER06', 

 'PE_MEMBER07', 

 'PE_MEMBER08', 

 'PE_MEMBER09', 

 'PE_MEMBER10', 

 'PE_MEMBER11', 

 'PE_MEMBER12', 

 'PE_MEMBER13', 

 'PE_MEMBER14', 

 'PE_MEMBER15', 

 'PE_MEMBER16', 

 'PE_MEMBER17', 

 'PE_MEMBER18', 

 'PE_MEMBER19:', 

 'PE_MEMBER20:', 

 'PE_MEMBER21:', 

 '', 

 '# For stochastic perturbed runs - added by Dhou and Wyang', 

 '--------------------------------------------------------', 

 '# ENS_SPS, logical control for application of stochastic ' 

 'perturbation scheme', 

 '# HH_START, start hour of forecast, and modified ' 'ADVANCECOUNT_SETUP', 

 '# HH_INCREASE and HH_FINAL are fcst hour increment and end ' 

 'hour of forecast', 

 '# ADVANCECOUNT_SETUP is an integer indicating the number of ' 

 'time steps between integration_start and the time when model ' 

 'state is saved for the _ini of the GEFS_Coupling, currently is ' 

 '0h.', 

 '', 

 'HH_INCREASE: 600', 

 'HH_FINAL: 600', 

 'HH_START: 0', 

 'ADVANCECOUNT_SETUP: 0', 

 '', 

 'ENS_SPS: .false.', 

 'HOUTASPS: 10000', 

 '', 

 '#ESMF_State_Namelist +++++++++++++++', 

 '', 

 'RUN_CONTINUE: .false.', 

 '', 

 '#', 

 'dt_int: 900', 

 'dt_num: 0', 

 'dt_den: 1', 

 f'start_year: {self.start_time.year}', 

 f'start_month: {self.start_time.month}', 

 f'start_day: {self.start_time.day}', 

 f'start_hour: {self.start_time.hour}', 

 f'start_minute: {self.start_time.minute}', 

 f'start_second: {self.start_time.second}', 

 f'nhours_fcst: {duration_hours:.0f}', 

 'restart: .false.', 

 f'nhours_fcst1: {duration_hours:.0f}', 

 'im: 192', 

 'jm: 94', 

 'global: .true.', 

 'nhours_dfini: 0', 

 'adiabatic: .false.', 

 'lsoil: 4', 

 'passive_tracer: .true.', 

 'dfilevs: 64', 

 'ldfiflto: .true.', 

 'num_tracers: 3', 

 'ldfi_grd: .false.', 

 'lwrtgrdcmp: .false.', 

 'nemsio_in: .false.', 

 '', 

 '', 

 '#jwstart added quilt', 

 '###############################', 

 '#### Specify the I/O tasks ####', 

 '###############################', 

 '', 

 '', 

 'quilting: .false. #For asynchronous ' 

 'quilting/history writes', 

 'read_groups: 0', 

 'read_tasks_per_group: 0', 

 'write_groups: 1', 

 'write_tasks_per_group: 3', 

 '', 

 'num_file: 3 #', 

 "filename_base: 'SIG.F' 'SFC.F' 'FLX.F'", 

 "file_io_form: 'bin4' 'bin4' 'bin4'", 

 "file_io: 'DEFERRED' 'DEFERRED' 'DEFERRED' " "'DEFERRED' #", 

 'write_dopost: .false. # True--> run do on ' 'quilt', 

 'post_gribversion: grib1 # True--> grib version for ' 

 'post output files', 

 'gocart_aer2post: .false.', 

 'write_nemsioflag: .TRUE. # True--> Write nemsio ' 

 'run history files', 

 'nfhout: 3', 

 'nfhout_hf: 1', 

 'nfhmax_hf: 0', 

 'nsout: 0', 

 '', 

 'io_recl: 100', 

 "io_position: ' '", 

 "io_action: 'WRITE'", 

 "io_delim: ' '", 

 "io_pad: ' '", 

 '', 

 '#jwend', 

 ] 

 )

implement nested run sequences

implement nested run sequences and arbitrary FORTRAN functions, as in this config:
https://github.com/NOAA-EMC/DATM-MOM6-CICE5/blob/ca3d21bd89f967e4c25953eaa3349829c739b528/parm/nems.configure.med_atm_ocn_ice.IN#L51-L70

# Run Sequence #
runSeq::
  @@[coupling_interval_slow_sec]
    MED MedPhase_prep_ocn
    MED -> OCN :remapMethod=redist
    OCN
    @@[coupling_interval_fast_sec]
      MED MedPhase_prep_ice
      MED -> ICE :remapMethod=redist
      ATM
      ICE
      ATM -> MED :remapMethod=redist
      ICE -> MED :remapMethod=redist
      MED MedPhase_atm_ocn_flux
      MED MedPhase_accum_fast
    @
    OCN -> MED :remapMethod=redist
    MED MedPhase_write_restart
  @
::

windows wheel test fails in PowerShell

build check fails on windows-latest due to unset environment variable; referenced in issue mtkennerly/dunamai-action#1

should duplicate entries be allowed in the run sequence?

when setting .sequence, the interface matches the first instance of the specified model / connection, and raises an error if a matching model / connection was already specified earlier in the passed sequence:

nemspy/nemspy/interface.py

Lines 100 to 145 in d1b45bb

 @sequence.setter 

 def sequence(self, sequence: [str]): 

 sequence_entries = [] 

 entries = {entry.sequence_entry: entry for entry in self.__sequence.sequence} 

 for entry in sequence: 

 if entry.upper() in entries: 

 sequence_entries.append(entries[entry.upper()]) 

 elif '->' in entry: 

 models = [model.strip() for model in entry.split('->')] 

 if len(models) == 2: 

 source, destination = models 

 for connection in self.__sequence.connections: 

 if len(connection.models) == 3 and None in connection.models: 

 connection_source, connection_destination = [ 

 model.model_type.value.upper() 

 for model in connection.models 

 if model is not None 

 ] 

 elif len(connection.models) == 2: 

 connection_source, connection_destination = [ 

 model.model_type.value.upper() for model in connection.models 

 ] 

 else: 

 continue 

 if ( 

 source == connection_source 

 and destination == connection_destination 

 ): 

 sequence_entries.append(connection) 

 break 

 else: 

 raise KeyError(f'"{entry}" not in {self.connections}') 

 elif len(models) == 3: 

 for mediation in self.__sequence.mediations: 

 if models == [ 

 model.model_type.value 

 for model in mediation.models 

 if model is not None 

 ]: 

 sequence_entries.append(mediation) 

 break 

 else: 

 raise KeyError(f'"{entry}" not in {self.connections}') 

 else: 

 raise KeyError(f'"{entry}" not in {self.sequence}') 

 self.__sequence.sequence = sequence_entries

This is potentially a problem if the same model or connection should be used more than once, such as

OCN
OCN -> ATM
ATM
ATM -> OCN
OCN
OCN -> WAV

@saeed-moghimi-noaa is this possible?

Sphinx configuration for NEMSpy

Sphinx RST files in `docs` directory

I generated the docs directory using the sphinx-quickstart command
./docs

`index.rst` (docs homepage)

this page includes the README file using m2r2, and then includes a hidden table of contents (so the sidebar includes these links)

nemspy/docs/source/index.rst

Lines 1 to 9 in 08b454c

 .. mdinclude:: ../../README.md 

 .. toctree:: 

 :hidden: 

 nemspy.interface 

 nemspy.model 

 nemspy.configuration 

 nemspy.utilities

example of a module page

Sphinx can use automodule, autoclass, and autofunction

nemspy/docs/source/nemspy.interface.rst

Lines 1 to 4 in 08b454c

 ``ModelingSystem`` Interface 

 ============================ 

 .. automodule:: nemspy.interface

dependencies

nemspy/setup.py

Line 42 in 08b454c

'documentation': ['dunamai', 'm2r2', 'sphinx', 'sphinx-rtd-theme'],

pip install -e .[documentation]

building locally

to build the website with Sphinx, run the following:

cd docs
make html

ReadTheDocs configuration

nemspy/.readthedocs.yml

Lines 1 to 20 in 08b454c

 # .readthedocs.yaml 

 # Read the Docs configuration file 

 # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 

 version: 2 

 build: 

 os: ubuntu-20.04 

 tools: 

 python: "3.9" 

 sphinx: 

 configuration: docs/source/conf.py 

 python: 

 install: 

 - method: pip 

 path: . 

 extra_requirements: 

 - documentation

	class ModelConfigurationFile(ConfigurationFile):
	name = 'model_configure'

	def __init__(self, start_time: datetime, duration: timedelta, sequence: RunSequence):
	self.start_time = start_time
	self.duration = duration
	super().__init__(sequence)

	def write(self, directory: PathLike, overwrite: bool = False):
	directory = ensure_directory(directory)
	filename = directory / self.name

	if filename.exists():
	LOGGER.warning(
	f'{"overwriting" if overwrite else "skipping"} ' f'existing file "{filename}"'
	)
	if not filename.exists() or overwrite:
	LOGGER.debug(f'writing model configuration to "{filename}"')
	with open(filename, 'w') as output_file:
	output_file.write(str(self))

	symbolic_link_filename = directory / 'atm_namelist.rc'
	try:
	symbolic_link_filename.symlink_to(filename)
	except Exception as error:
	LOGGER.warning(f'could not create symbolic link: {error}')
	with open(symbolic_link_filename, 'w') as output_file:
	output_file.write(str(self))

	def __str__(self) -> str:
	duration_hours = round(self.duration / timedelta(hours=1))
	return '\n'.join(
	[
	'core: gfs',
	'print_esmf: .true.',
	'',
	'nhours_dfini=0',
	'',
	'#nam_atm +++++++++++++++++++++++++++',
	'nlunit: 35',
	'deltim: 900.0',
	'fhrot: 0',
	'namelist: atm_namelist',
	'total_member: 1',
	'grib_input: 0',
	f'PE_MEMBER01: {self.sequence.processors}',
	'PE_MEMBER02',
	'PE_MEMBER03',
	'PE_MEMBER04',
	'PE_MEMBER05',
	'PE_MEMBER06',
	'PE_MEMBER07',
	'PE_MEMBER08',
	'PE_MEMBER09',
	'PE_MEMBER10',
	'PE_MEMBER11',
	'PE_MEMBER12',
	'PE_MEMBER13',
	'PE_MEMBER14',
	'PE_MEMBER15',
	'PE_MEMBER16',
	'PE_MEMBER17',
	'PE_MEMBER18',
	'PE_MEMBER19:',
	'PE_MEMBER20:',
	'PE_MEMBER21:',
	'',
	'# For stochastic perturbed runs - added by Dhou and Wyang',
	'--------------------------------------------------------',
	'# ENS_SPS, logical control for application of stochastic '
	'perturbation scheme',
	'# HH_START, start hour of forecast, and modified ' 'ADVANCECOUNT_SETUP',
	'# HH_INCREASE and HH_FINAL are fcst hour increment and end '
	'hour of forecast',
	'# ADVANCECOUNT_SETUP is an integer indicating the number of '
	'time steps between integration_start and the time when model '
	'state is saved for the _ini of the GEFS_Coupling, currently is '
	'0h.',
	'',
	'HH_INCREASE: 600',
	'HH_FINAL: 600',
	'HH_START: 0',
	'ADVANCECOUNT_SETUP: 0',
	'',
	'ENS_SPS: .false.',
	'HOUTASPS: 10000',
	'',
	'#ESMF_State_Namelist +++++++++++++++',
	'',
	'RUN_CONTINUE: .false.',
	'',
	'#',
	'dt_int: 900',
	'dt_num: 0',
	'dt_den: 1',
	f'start_year: {self.start_time.year}',
	f'start_month: {self.start_time.month}',
	f'start_day: {self.start_time.day}',
	f'start_hour: {self.start_time.hour}',
	f'start_minute: {self.start_time.minute}',
	f'start_second: {self.start_time.second}',
	f'nhours_fcst: {duration_hours:.0f}',
	'restart: .false.',
	f'nhours_fcst1: {duration_hours:.0f}',
	'im: 192',
	'jm: 94',
	'global: .true.',
	'nhours_dfini: 0',
	'adiabatic: .false.',
	'lsoil: 4',
	'passive_tracer: .true.',
	'dfilevs: 64',
	'ldfiflto: .true.',
	'num_tracers: 3',
	'ldfi_grd: .false.',
	'lwrtgrdcmp: .false.',
	'nemsio_in: .false.',
	'',
	'',
	'#jwstart added quilt',
	'###############################',
	'#### Specify the I/O tasks ####',
	'###############################',
	'',
	'',
	'quilting: .false. #For asynchronous '
	'quilting/history writes',
	'read_groups: 0',
	'read_tasks_per_group: 0',
	'write_groups: 1',
	'write_tasks_per_group: 3',
	'',
	'num_file: 3 #',
	"filename_base: 'SIG.F' 'SFC.F' 'FLX.F'",
	"file_io_form: 'bin4' 'bin4' 'bin4'",
	"file_io: 'DEFERRED' 'DEFERRED' 'DEFERRED' " "'DEFERRED' #",
	'write_dopost: .false. # True--> run do on ' 'quilt',
	'post_gribversion: grib1 # True--> grib version for '
	'post output files',
	'gocart_aer2post: .false.',
	'write_nemsioflag: .TRUE. # True--> Write nemsio '
	'run history files',
	'nfhout: 3',
	'nfhout_hf: 1',
	'nfhmax_hf: 0',
	'nsout: 0',
	'',
	'io_recl: 100',
	"io_position: ' '",
	"io_action: 'WRITE'",
	"io_delim: ' '",
	"io_pad: ' '",
	'',
	'#jwend',
	]
	)

	@sequence.setter
	def sequence(self, sequence: [str]):
	sequence_entries = []
	entries = {entry.sequence_entry: entry for entry in self.__sequence.sequence}
	for entry in sequence:
	if entry.upper() in entries:
	sequence_entries.append(entries[entry.upper()])
	elif '->' in entry:
	models = [model.strip() for model in entry.split('->')]
	if len(models) == 2:
	source, destination = models
	for connection in self.__sequence.connections:
	if len(connection.models) == 3 and None in connection.models:
	connection_source, connection_destination = [
	model.model_type.value.upper()
	for model in connection.models
	if model is not None
	]
	elif len(connection.models) == 2:
	connection_source, connection_destination = [
	model.model_type.value.upper() for model in connection.models
	]
	else:
	continue
	if (
	source == connection_source
	and destination == connection_destination
	):
	sequence_entries.append(connection)
	break
	else:
	raise KeyError(f'"{entry}" not in {self.connections}')
	elif len(models) == 3:
	for mediation in self.__sequence.mediations:
	if models == [
	model.model_type.value
	for model in mediation.models
	if model is not None
	]:
	sequence_entries.append(mediation)
	break
	else:
	raise KeyError(f'"{entry}" not in {self.connections}')
	else:
	raise KeyError(f'"{entry}" not in {self.sequence}')
	self.__sequence.sequence = sequence_entries

	.. mdinclude:: ../../README.md

	.. toctree::
	:hidden:

	nemspy.interface
	nemspy.model
	nemspy.configuration
	nemspy.utilities

	``ModelingSystem`` Interface
	============================

	.. automodule:: nemspy.interface

	# .readthedocs.yaml
	# Read the Docs configuration file
	# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details

	version: 2

	build:
	os: ubuntu-20.04
	tools:
	python: "3.9"

	sphinx:
	configuration: docs/source/conf.py

	python:
	install:
	- method: pip
	path: .
	extra_requirements:
	- documentation