205 lines
12 KiB
Python
205 lines
12 KiB
Python
import os
|
|
import click
|
|
|
|
from sdi_checker.app_cli import display_check
|
|
|
|
|
|
@click.command(name='csw')
|
|
@click.argument('url', nargs=-1) # help = the server to target (full URL, e.g. https://sdi.georchestra.org/geoserver/wms)
|
|
@click.option('--inspire', '-i', show_default=True, default='flexible', type=click.Choice(['flexible', 'strict'], case_sensitive=False), help='indicates if the checks should be strict or flexible, default to flexible')
|
|
@click.option('--geoserver-to-check', '-g', default=None, type=str, multiple=True, help="space-separated list of geoserver hostname to check in CSW mode with inspire strict option activated. Ex: sdi.georchestra.org")
|
|
@click.option('--disable-ssl-verification', '-ssl', is_flag=True, type=bool, help="disable certificate verification")
|
|
@click.option('--only-err', '-e', is_flag=True, type=bool, help="only display errors, no summary informations will be displayed")
|
|
@click.option('--xunit', '-x', is_flag=True, type=bool, help="generate a XML xunit result report")
|
|
@click.option('--xunit-output', '-xo', default="xunit.xml", show_default=True, type=str, help="name of the xunit report file, defaults to ./xunit.xml")
|
|
@click.option('--log-to-file', '-l', default='', type=str, help="if a file path is specified, log output to this file, not stdout")
|
|
@click.option('--timeout', '-t', default=0, type=int, help="specify a timeout for request to external service.")
|
|
@click.option('--output', '-o', type=click.Choice(['json', 'txt', 'screen', 'csv', 'db'], case_sensitive=False), default='screen', help="output format")
|
|
@click.pass_obj
|
|
def run(app, url, inspire, geoserver_to_check, disable_ssl_verification, only_err, xunit, xunit_output, log_to_file, timeout, output):
|
|
"""
|
|
> master add [TEXT] [--template TPL] [--file FILE] [--edit] [--repository REPOSITORY] [--login LOGIN] [--password PASSWORD]
|
|
"""
|
|
on_csw(app, url, inspire, geoserver_to_check, disable_ssl_verification, only_err, xunit, xunit_output, log_to_file, timeout, output)
|
|
|
|
|
|
def on_csw(app, url, inspire, geoserver_to_check, disable_ssl_verification, only_err, xunit, xunit_output, log_to_file, timeout, output):
|
|
"""
|
|
> master add [TEXT] [--template TPL] [--file FILE] [--edit] (=> open edit window)
|
|
"""
|
|
# if helpers.check_repository_connect(app, repository, login, password):
|
|
# # Backup main directory
|
|
# app.backup_files()
|
|
|
|
# text = '\n====\n'.join(text) if text else None
|
|
# # TODO: check use of --edit, --file and --template parameters
|
|
# if text is None or edit or file or template:
|
|
# path = None
|
|
# if file:
|
|
# path = os.path.split(file)
|
|
# if template:
|
|
# path = [app.templates_directory, template + '.txt']
|
|
# text = app.get_template(path[1], path[0]) if path is not None else text
|
|
|
|
# text = click.edit(text) or None
|
|
|
|
# if text is not None:
|
|
|
|
# insert_result = app.add_items(text)
|
|
# save_result = app.save_items_to_files(insert_result['items'])
|
|
|
|
# # Display clean report
|
|
# display.display_save_report(app, save_result['total_items'], save_result['total_bad_items'], save_result['total_good_items'], save_result['total_wrote_items'])
|
|
# app.echo()
|
|
# app.echo('{nb_added_items} items saved (total: {nb_total_new})'.format(nb_added_items=insert_result['nb_added_items'], nb_total_new=insert_result['nb_total_new']))
|
|
# app.echo('Added items MD5: {md5}'.format(md5=insert_result['md5']))
|
|
# if insert_result['nb_exist_items'] > 0:
|
|
# app.echo('{nb_exist_items} item(s) already exist(s)'.format(nb_exist_items=insert_result['nb_exist_items']))
|
|
# app.echo()
|
|
|
|
# app.rprompt = '{nb_added_items} item(s) saved'.format(nb_added_items=insert_result['nb_added_items'])
|
|
# else:
|
|
# app.echo('No item added.')
|
|
# app.echo()
|
|
|
|
if url is not None:
|
|
total_mds = 0
|
|
geoserver_services = CachedOwsServices(creds, disable_ssl=args.disable_ssl_verification, timeout=request_timeout)
|
|
try:
|
|
csw_q = CSWQuerier(args.server, credentials=creds, cached_ows_services=geoserver_services, logger=logger, timeout=request_timeout)
|
|
except ServiceException as e:
|
|
logger.debug(e, exc_info=True)
|
|
logger.fatal("Unable to query the remote CSW:\nError: %s\nPlease check the CSW url", e)
|
|
sys.exit(1)
|
|
errors = []
|
|
reporting = []
|
|
if args.inspire == "strict":
|
|
# Step 1: get all data metadata
|
|
datamd = csw_q.get_all_records(constraints=[And([csw_q.is_dataset, csw_q.non_harvested])])
|
|
# Step 2: maps data metadatas to service MDs
|
|
servicesmd = csw_q.get_all_records(constraints=[And([csw_q.is_service, csw_q.non_harvested])])
|
|
data_to_service_map = {}
|
|
for uuid, md in servicesmd.items():
|
|
for oon in md.identificationinfo[0].operateson:
|
|
if data_to_service_map.get(oon['uuidref']) is None:
|
|
data_to_service_map[oon['uuidref']] = [uuid]
|
|
else:
|
|
data_to_service_map[oon['uuidref']] = data_to_service_map[oon['uuidref']] + [uuid]
|
|
|
|
# Step 3: on each data md, get the service md, and the underlying service URL
|
|
#for uuid, md in enumerate(datamd):
|
|
for mdd_uuid, mdd in datamd.items():
|
|
# Note: this won't count the service metadata in the end, only the MDD that trigger a
|
|
# check onto a service MD.
|
|
total_mds += 1
|
|
|
|
if data_to_service_map.get(mdd_uuid) is None:
|
|
# TODO file an issue if the dataMd has no ServiceMd linked to ?
|
|
if len([x for x in reporting if x['uuid'] == mdd_uuid]) == 0:
|
|
reporting.append({ 'classname': 'CSW', 'name': mdd.identification.title, 'uuid': mdd_uuid,
|
|
'time': '0', 'error': None })
|
|
continue
|
|
# step 4: check the layer existence using the service URL
|
|
for sce_uuid in data_to_service_map[mdd_uuid]:
|
|
try:
|
|
mds = servicesmd[sce_uuid]
|
|
mdd = datamd[mdd_uuid]
|
|
csw_q.check_service_md(mds, mdd, geoserver_to_check=args.geoserver_to_check if
|
|
args.geoserver_to_check is not None else [])
|
|
# No issue so far ?
|
|
# since a MDD can reference several service metadata, consider
|
|
# the MDD as passing tests only once (avoid adding several times the same MDD
|
|
# to the array). It must be very unlikely to have several MDS anyway.
|
|
if len([x for x in reporting if x['uuid'] == mdd_uuid]) == 0:
|
|
reporting.append({ 'classname': 'CSW', 'name': mdd.title, 'uuid': mdd_uuid,
|
|
'time': '0', 'error': None })
|
|
except Inconsistency as e:
|
|
logger.debug(e, exc_info=True)
|
|
logger.error(e)
|
|
errors.append(e)
|
|
# Same as above: only adding the errored MDD once
|
|
if len([x for x in reporting if x['uuid'] == mdd_uuid]) == 0:
|
|
reporting.append({ 'classname': 'CSW', 'name': mdd.title, 'uuid': mdd_uuid,
|
|
'time': '0', 'error': e })
|
|
|
|
elif args.inspire == "flexible":
|
|
global_idx = 0
|
|
csw_q.start = 0
|
|
while True:
|
|
|
|
res = csw_q.get_dataset_records(constraints=[csw_q.non_harvested])
|
|
|
|
total_mds += len(res)
|
|
for idx, uuid in enumerate(res):
|
|
current_md = res[uuid]
|
|
logger.info("#%d\n UUID : %s\n %s", global_idx, uuid, current_md.title)
|
|
wms_found = False
|
|
wfs_found = False
|
|
|
|
for uri in csw_q.get_md(uuid).uris:
|
|
from_wms = False
|
|
try:
|
|
if uri["protocol"] == "OGC:WMS":
|
|
wms_found = True
|
|
from_wms = True
|
|
# TODO: use the geoserver_to_check option ?
|
|
geoserver_services.checkWmsLayer(uri["url"], uri["name"])
|
|
|
|
logger.debug("\tURI OK : %s %s %s", uri["protocol"], uri['url'], uri['name'])
|
|
logger.info(" WMS url: OK")
|
|
elif uri["protocol"] == "OGC:WFS":
|
|
wfs_found = True
|
|
# TODO: same remark
|
|
geoserver_services.checkWfsLayer(uri["url"], uri["name"])
|
|
logger.debug("\tURI OK : %s %s %s", uri["protocol"], uri['url'], uri['name'])
|
|
logger.info(" WFS url: OK")
|
|
else:
|
|
logger.debug("\tSkipping URI : %s %s %s", uri["protocol"], uri['url'], uri['name'])
|
|
except Exception as ex:
|
|
if isinstance(ex, GnToGsLayerNotFoundInconsistency) or \
|
|
isinstance(ex, GnToGsInvalidCapabilitiesUrl) or \
|
|
isinstance(ex,GnToGsOtherError):
|
|
ex.set_md_uuid(uuid)
|
|
errors.append(ex)
|
|
else:
|
|
# morph encountered error in to an "other error"
|
|
exc = GnToGsOtherError(uri['url'], uri['name'], ex)
|
|
exc.set_md_uuid(uuid)
|
|
errors.append(exc)
|
|
logger.debug("\t /!\\ ---> Cannot find Layer ON GS : %s %s %s %s %s",
|
|
uuid, uri['protocol'], uri['url'], uri['name'], ex)
|
|
logger.info(" %s url: KO: %s: %s" % ("WMS" if from_wms else "WFS",
|
|
uri['url'], str(errors[-1])))
|
|
# in both cases, add the MDD in the reporting array
|
|
if len([x for x in reporting if x['uuid'] == uuid]) == 0:
|
|
reporting.append({ 'classname': 'CSW', 'name': current_md.title, 'uuid': uuid,
|
|
'time': '0', 'error': ex })
|
|
|
|
if not wms_found:
|
|
logger.info(" WMS url: KO: No wms url found in the metadata")
|
|
err = GnToGsNoOGCWmsDefined(uuid)
|
|
errors.append(err)
|
|
reporting.append({ 'classname': 'CSW', 'name': current_md.title, 'uuid': uuid,
|
|
'time': '0', 'error': err })
|
|
|
|
if not wfs_found:
|
|
logger.info(" WFS url: KO: No wfs url found in the metadata")
|
|
err = GnToGsNoOGCWfsDefined(uuid)
|
|
errors.append(err)
|
|
reporting.append({ 'classname': 'CSW', 'name': current_md.title, 'uuid': uuid,
|
|
'time': '0', 'error': err })
|
|
if wms_found and wfs_found:
|
|
reporting.append({ 'classname': 'CSW', 'name': current_md.title, 'uuid': uuid,
|
|
'time': '0', 'error': None })
|
|
|
|
logger.info("")
|
|
# end of current md
|
|
global_idx += 1
|
|
|
|
if csw_q.start > csw_q.csw.results['matches']:
|
|
break
|
|
|
|
print_csw_report(errors, total_mds)
|
|
if args.xunit:
|
|
generate_csw_xunit_layers_status(reporting, args.xunit_output)
|
|
|