Source code for ppcpy.interface.picassoProc

#from ..misc import *
import datetime
import re
import numpy as np
import logging
from pathlib import Path
import ppcpy.misc.pollyChannelTags as pollyChannelTags
import ppcpy.preprocess.pollyPreprocess as pollyPreprocess
import ppcpy.qc.pollySaturationDetect as pollySaturationDetect
import ppcpy.qc.transCor as transCor
import ppcpy.qc.overlapEst as overlapEst
import ppcpy.qc.overlapCor as overlapCor
import ppcpy.qc.qualityMask as qualityMask

import ppcpy.calibration.select as select
import ppcpy.calibration.polarization as polarization
import ppcpy.cloudmask.cloudscreen as cloudscreen
import ppcpy.cloudmask.profilesegment as profilesegment
import ppcpy.preprocess.profiles as preprocprofiles
import ppcpy.io.readMeteo as readMeteo
import ppcpy.misc.molecular as molecular
import ppcpy.calibration.watervapor as watervapor
import ppcpy.calibration.rayleighfit as rayleighfit
import ppcpy.retrievals.klettfernald as klettfernald
import ppcpy.retrievals.raman as raman
import ppcpy.retrievals.depolarization as depolarization 
import ppcpy.retrievals.angstroem as angstroem 
import ppcpy.calibration.lidarconstant as lidarconstant

import ppcpy.retrievals.highres as highres
import ppcpy.retrievals.quasiV1 as quasiV1
import ppcpy.retrievals.quasiV2 as quasiV2
import ppcpy.retrievals.quasi as quasi

import ppcpy.io.sql_interaction as sql_db


[docs] class PicassoProc: """Picasso Processor. This class is responsible for perfoming the processing of PollyXT data. """ counter = 0 def __init__(self, rawdata_dict:dict, polly_config_dict:dict, picasso_config_dict:dict): """Initialize the PicassoProc object. Parameters ---------- rawdata_dict : dict The dict returned by readPollyRawData.readPollyRawData(filename=rawfile). polly_config_dict : dict The configuration specific to the specific polly loadConfigs.loadPollyConfig(polly_config_file_fullname, polly_default_config_file). picasso_config_dict : dict The general picasso config loadConfigs.loadPicassoConfig(args.picasso_config_file,picasso_default_config_file). Attributes ---------- self.rawfile : str Path to level0-file. self.rawdata_dict : dict The input parameter `rawdata_dict`. self.polly_config_dict : dict The input parameter `polly_config_dict`. self.picasso_config_dict : dict The input parameter `picasso_config_dict`. self.device : str Name of pollyXT device. self.location : ... Location of PollyXT device during the measurement. self.date : str Measurement date. self.num_of_channels : int Number of measurment channels. self.num_of_profiles : int Number of profiles in the measurment (time-dimension). self.retrievals_highres : dict Dictionary to store high resolution time, height data. self.retrievals_profile : dict Dictionary to store profile date. self.retrievals_profile['avail_optical_profiles'] : list Availabele optical profiles. self.pol_cali : dict Dictionary to store polarization calibration constants. self.LC : dict Dictionary to store lidar calibration constants. self.wv_cali : dict Dictionary to store water vapor calibration constants. Notes ----- - The `polly_default_dict` is not longer available as a separate variable, but is included into the `polly_config_dict`. """ type(self).counter += 1 self.rawfile = rawdata_dict['filename_path'] self.rawdata_dict = rawdata_dict self.polly_config_dict = polly_config_dict self.picasso_config_dict = picasso_config_dict self.device = self.polly_config_dict['name'] self.location = self.polly_config_dict['site'] self.date = self.mdate_filename() self.num_of_channels = len(self.rawdata_dict['measurement_shots']['var_data'][0]) self.num_of_profiles = self.rawdata_dict['raw_signal']['var_data'].shape[0] self.retrievals_highres = {} self.retrievals_profile = {} self.retrievals_profile['avail_optical_profiles'] = [] self.pol_cali = {} self.LC = {} self.wv_cali = {}
[docs] def mdate_filename(self) -> str: """Get the date from filename in YYYYMMDD. Returns ------- str Measurement date. """ filename = self.rawdata_dict['filename'] mdate = re.split(r'_', filename)[0:3] YYYY = mdate[0] MM = mdate[1] DD = mdate[2] return f"{YYYY}{MM}{DD}"
[docs] def gf(self, wavelength:float|int|str, meth:str, telescope:str) -> np.ndarray: """Get flag shorthand. i.e., the following two calls are equivalent ``` data_cube.flag_532_total_FR data_cube.gf(532, 'total', 'FR') ``` where the pattern `{wavelength}_{total|cross|parallel|rr}_{NR|FR|DFOV}` from https://github.com/PollyNET/Pollynet_Processing_Chain/issues/303 is obeyed Parameters ---------- wavelength : float or int or str Wavelength tag. meth : str Method type. telescope : str Telescope type. Returns ------- ndarray With bool flag """ return getattr(self, f'flag_{wavelength}_{meth}_{telescope}', False)
# def msite(self): # #msite = f"measurement site: {self.rawdata_dict['global_attributes']['location']}" # msite = self.polly_config_dict['site'] # logging.info(f'measurement site: {msite}') # return msite # # # def device(self): # device = self.polly_config_dict['name'] # logging.info(f'measurement device: {device}') # return device # # # def mdate(self): # mdate = self.mdate_filename() # logging.info(f'measuremnt date: {mdate}') # return mdate
[docs] def mdate_infile(self) -> str: """First date in file as string. Returns ------- str Measurement date. """ mdate_infilename = self.rawdata_dict['measurement_time']['var_data'][0][0] return f"{mdate_infilename}"
[docs] def check_for_correct_mshots(self) -> np.ndarray: """Check if mshots are more than 1.1 * laser_prf * deltatime or smaller 0. Returns ------- condition_check_matrix : ndarray Boolean array. True for elements where the condition above holds, otherwise False. """ laser_rep_rate = self.rawdata_dict['laser_rep_rate']['var_data'] mShotsPerPrf = laser_rep_rate * self.polly_config_dict['deltaT'] mShots = self.rawdata_dict['measurement_shots']['var_data'] # Check for values > 1.1*mShotsPerPrf or <= 0 condition_check_matrix = (mShots > mShotsPerPrf*1.1) | (mShots <= 0) return condition_check_matrix
[docs] def filter_or_correct_false_mshots(self): """Filter or correct the mshots (currently only logging). .. TODO:: that might be covered via the mcps conversion --> How exactly??? Attributes ---------- self.rawdata_dict : dict With filtered or corrected enteries for timestamps where `data_cube.check_for_correct_mshots()=True`. Notes ----- - Now includes a first try for the function. I Do not know if it actually is needed yet. .. TODO:: Not yet crosschecked with the matlab version. ** History ** - 2026-06-24: Translated to python """ ## Extract the necesary information laser_rep_rate = self.rawdata_dict['laser_rep_rate']['var_data'] mShotsPerPrf = laser_rep_rate * self.polly_config_dict['deltaT'] mShots = self.rawdata_dict['measurement_shots']['var_data'] mTime = self.rawdata_dict['measurement_time']['var_data'] rawSig = self.rawdata_dict['raw_signal']['var_data'] condition_check_matrix = self.check_for_correct_mshots() condition_check_matrix = np.any(condition_check_matrix, axis=0) ## Filter out timesteps with faulty measurement shots if self.polly_config_dict['flagFilterFalseMShots']: logging.info('filtering false mshots') if np.sum(~condition_check_matrix) == 0: logging.critical('No profile with mshots < 1e6 and mshots > 0 was found Please take a look inside level0 file.') # raise ValueError('No profile with mshots < 1e6 and mshots > 0 was found Please take a look inside level0 file.') return rawSig = rawSig[~condition_check_matrix] mShots = mShots[~condition_check_matrix] mTime = mTime[~condition_check_matrix] if 'depol_cal_angle' in self.rawdata_dict: self.rawdata_dict['depol_cal_angle']['var_data'] \ = self.rawdata_dict['depol_cal_angle']['var_data'][~condition_check_matrix] ## Correct timesteps with faulty measurement shots elif self.polly_config_dict['flagCorrectFalseMShots']: logging.info('correcting false mshots') mShots[condition_check_matrix] = mShotsPerPrf # .. TODO:: In the matlab vesion this also does a fix to mTime. I do not know if this is necesary yet. ## Overwrite the filtered / corrected information self.rawdata_dict['measurement_shots']['var_data'] = mShots self.rawdata_dict['measurement_time']['var_data'] = mTime self.rawdata_dict['raw_signal']['var_data'] = rawSig
[docs] def mdate_consistency(self) -> bool: """Check mdate consistency. Returns ------- bool True, if measurement date of file name equals that of in the file, otherwise False. """ if self.mdate_filename() == self.mdate_infile(): logging.info('... date in nc-file equals date of filename') return True else: logging.warning('... date in nc-file differs from date of filename') return False
[docs] def reset_date_infile(self): """Correct the date in the file. Attributes ---------- self.rawdata_dict['measurement_time']['var_data'] : ndarray Updated .... Notes ----- .. TODO :: Finish docstring. """ logging.info('date consistency-check... ') if self.mdate_consistency() == False: logging.info('date in nc-file will be replaced with date of filename.') np_array = np.array(self.rawdata_dict['measurement_time']['var_data']) ## converting to numpy-array for easier numpy-operations mdate = self.mdate_filename() np_array[:, 0] = mdate ## assign new date value to the whole first column of the 2d-numpy-array self.rawdata_dict['measurement_time']['var_data'] = np_array
[docs] def setChannelTags(self): """Set the channel tags. they are stored as dictionary in ``` data_cube.channel_dict ``` and as list in ``` data_cube.retrievals_highres['channel'] data_cube.polly_config_dict['channelTags'] ``` as an array of boolean flags in ``` data_cube.flags ``` as flags per channel in ``` data_cube.flag_355_total_FR ``` Attributes ---------- self.retrievals_highres['channel'] : list Updated channel tags. self.polly_config_dict['channelTags'] : list updated channel tags. self.flag_355_total_FR : ndarray True if the channel is `355nm total FR` else False. self.flag_355_cross_FR : ndarray True if the channel is `355nm cross FR` else False. self.flag_355_parallel_FR : ndarray True if the channel is `355nm parallel FR` else False. self.flag_355_total_NR : ndarray True if the channel is `355nm total NR` else False. self.flag_387_total_FR : ndarray True if the channel is `387nm total FR` else False. self.flag_387_total_NR : ndarray True if the channel is `387nm total NR` else False. self.flag_407_total_FR : ndarray True if the channel is `407nm total FR` else False. self.flag_407_total_NR : ndarray True if the channel is `407nm total NR` else False. self.flag_532_total_FR : ndarray True if the channel is `532nm total FR` else False. self.flag_532_cross_FR : ndarray True if the channel is `532nm cross FR` else False. self.flag_532_parallel_FR : ndarray True if the channel is `532nm parallel FR` else False. self.flag_532_total_NR : ndarray True if the channel is `532nm total NR` else False. self.flag_532_cross_DFOV : ndarray True if the channel is `532nm cross DFOV` else False. self.flag_532_rr_FR : ndarray True if the channel is `532nm rr FR` else False. self.flag_607_total_FR : ndarray True if the channel is `607nm total FR` else False. self.flag_607_total_NR : ndarray True if the channel is `607nm total NR` else False. self.flag_1058_total_FR : ndarray True if the channel is `1058nm total FR` else False. self.flag_1064_total_FR : ndarray True if the channel is `1064nm total FR` else False. self.flag_1064_cross_FR : ndarray True if the channel is `1064nm cross FR` else False. self.flag_1064_total_NR : ndarray True if the channel is `1064nm total NR` else False. """ channelTags = pollyChannelTags.pollyChannelTags( self.polly_config_dict['channelTag'], # TODO key: channelTags vs channelTag??? flagFarRange=self.polly_config_dict['isFR'], flagNearRange=self.polly_config_dict['isNR'], flagRotRaman=self.polly_config_dict['isRR'], flagTotal=self.polly_config_dict['isTot'], flagCross=self.polly_config_dict['isCross'], flagParallel=self.polly_config_dict['isParallel'], flag355nm=self.polly_config_dict['is355nm'], flag387nm=self.polly_config_dict['is387nm'], flag407nm=self.polly_config_dict['is407nm'], flag532nm=self.polly_config_dict['is532nm'], flag607nm=self.polly_config_dict['is607nm'], flag1064nm=self.polly_config_dict['is1064nm'], flagDFOV=self.polly_config_dict['isDFOV'], flag460nm=self.polly_config_dict['is460nm'], flag353nm=self.polly_config_dict['is353nm'], flag530nm=self.polly_config_dict['is530nm'], flag1058nm=self.polly_config_dict['is1058nm'], ) ## Remove "none" channels channelTags, self.polly_config_dict = pollyChannelTags.polly_config_channel_corrections( chTagsOut_ls=channelTags, polly_config_dict=self.polly_config_dict ) self.retrievals_highres['channel'] = channelTags self.polly_config_dict['channelTags'] = channelTags self.channel_dict = {i: item for i, item in enumerate(channelTags)} logging.info(f'ChannelTags: {channelTags}') ## Construct channel flags channelFlags = pollyChannelTags.pollyChannelFlags( channel_dict_length=len(self.channel_dict), flagFarRange=self.polly_config_dict['isFR'], flagNearRange=self.polly_config_dict['isNR'], flagRotRaman=self.polly_config_dict['isRR'], flagTotal=self.polly_config_dict['isTot'], flagCross=self.polly_config_dict['isCross'], flagParallel=self.polly_config_dict['isParallel'], flag355nm=self.polly_config_dict['is355nm'], flag387nm=self.polly_config_dict['is387nm'], flag407nm=self.polly_config_dict['is407nm'], flag532nm=self.polly_config_dict['is532nm'], flag607nm=self.polly_config_dict['is607nm'], flag1064nm=self.polly_config_dict['is1064nm'], flagDFOV=self.polly_config_dict['isDFOV'], flag460nm=self.polly_config_dict['is460nm'], flag353nm=self.polly_config_dict['is353nm'], flag530nm=self.polly_config_dict['is530nm'], flag1058nm=self.polly_config_dict['is1058nm'], ) ## Store channel flags self.flags = list(channelFlags.values()) self.flag_353_total_FR = channelFlags['far-range total 353 nm'] self.flag_355_total_FR = channelFlags['far-range total 355 nm'] self.flag_355_cross_FR = channelFlags['far-range cross 355 nm'] self.flag_355_parallel_FR = channelFlags['far-range parallel 355 nm'] self.flag_355_total_NR = channelFlags['near-range total 355 nm'] self.flag_387_total_FR = channelFlags['far-range 387 nm'] self.flag_387_total_NR = channelFlags['near-range 387 nm'] self.flag_407_total_FR = channelFlags['far-range 407 nm'] self.flag_407_total_NR = channelFlags['near-range 407 nm'] self.flag_460_total_FR = channelFlags['far-range 460 nm'] self.flag_530_total_FR = channelFlags['far-range total 530 nm'] self.flag_532_total_FR = channelFlags['far-range total 532 nm'] self.flag_532_cross_FR = channelFlags['far-range cross 532 nm'] self.flag_532_parallel_FR = channelFlags['far-range parallel 532 nm'] self.flag_532_total_NR = channelFlags['near-range total 532 nm'] self.flag_532_cross_DFOV = channelFlags['near-range cross 532 nm'] self.flag_607_total_FR = channelFlags['far-range 607 nm'] self.flag_607_total_NR = channelFlags['near-range 607 nm'] self.flag_1058_total_FR = channelFlags['far-range total 1058 nm'] self.flag_1064_total_FR = channelFlags['far-range total 1064 nm'] self.flag_1064_cross_FR = channelFlags['far-range cross 1064 nm'] self.flag_1064_parallel_FR = channelFlags['far-range parallel 1064 nm'] self.flag_1064_total_NR = channelFlags['near-range total 1064 nm'] self.flag_unknown = channelFlags['unknown'] logging.debug(f"channelFlags: {channelFlags}") if np.sum(self.flag_unknown) > 0: logging.warning(f"Unknown channels: {np.asarray(self.retrievals_highres['channel'])[self.flag_unknown]}")
[docs] def preprocessing(self, collect_debug:bool=False): """Preprocessing of Lidar data. Includes the followin processes in order: 1. Deadtime correction 2. Background correction 3. First-bin shift 4. Mask for low-SNR 5. Mask bins with laser shutter on 6. Mask bins with fog 7. Mask for depolarization-calibration process 8. Range correction. Parameters ---------- collect_debug : bool, optional If true, collects debug information. Default is False. Attributes ---------- self.retrievals_highres : dict With preprocessed lidar data. Keys ---- mShots : ndarray Number of the laser shots for each profile. sigDTCor : ndarray Dead time corrected signal [photon count]. BG : ndarray Background. sigBGCor : ndarray Backgound corrected signal [photon count]. range : ndarray Height above ground at zenith angle [m]. height : ndarray Height above ground [m]. alt : ndarray Altitude [m]. time : list Measurement time in unix time. time64 : ndarray Measurement time in numpy.datetime. SNR : ndarray Signal to noise ratio. lowSNRMask : ndarray True if SNR is less SNRmin. Otherwise False. shutterOnMask : ndarray True if at timesteps where the shutters was on. Otherwise False. fogMask : ndarray If it is foggy which means the signal will be very weak, fogMask will be set True. Otherwise, False. mask607Off : ndarray Mask of PMT on/off status at 607 nm channel. mask387Off : ndarray Mask of PMT on/off status at 387 nm channel. mask407Off : ndarray Mask of PMT on/off status at 407 nm channel. mask355RROff : ndarray Mask of PMT on/off status at 355 nm rotational Raman channel. mask532RROff : ndarray Mask of PMT on/off status at 532 nm rotational Raman channel. mask1064RROff : ndarray Mask of PMT on/off status at 1064 nm rotational Raman channel. depCal_P_Ang_time_start : list Time for the first profile with valid positive angle depolarization calibration. depCal_P_Ang_time_end : list Time for the last profile with valid positive angle depolarization calibration. depCal_N_Ang_time_start : list Time for the first profile with valid negative angle depolarization calibration. depCal_N_Ang_time_end : list Time for the last profile with valid negative angle depolarization calibration. maskDepCal : ndarray If polly was doing polarization calibration, depCalMask is set True. Otherwise, False. RCS : ndarray Range Corrected signal [MCPS]. PCR_slice : ndarray Background corrected signal in photon count rate [MCPS]. Notes ----- .. TODO:: - This is just a first draft for a docstring. Improve it. There is more processes and outputs of the function. - Also go over the inputs and see if all of them are actually needed. """ logging.info("Preprocessing ...") preproc_dict = pollyPreprocess.pollyPreprocess( self.rawdata_dict, deltaT=self.polly_config_dict['deltaT'], flagForceMeasTime=self.polly_config_dict['flagForceMeasTime'], maxHeightBin=self.polly_config_dict['max_height_bin'], firstBinIndex=self.polly_config_dict['first_range_gate_indx'], firstBinHeight=self.polly_config_dict['first_range_gate_height'], pollyType=self.polly_config_dict['name'], flagDeadTimeCorrection=self.polly_config_dict['flagDTCor'], deadtimeCorrectionMode=self.polly_config_dict['dtCorMode'], deadtimeParams=self.polly_config_dict['dt'], flagSigTempCor=self.polly_config_dict['flagSigTempCor'], tempCorFunc=self.polly_config_dict['tempCorFunc'], meteorDataSource=self.polly_config_dict['meteorDataSource'], gdas1Site=self.polly_config_dict['gdas1Site'], gdas1_folder=self.picasso_config_dict['gdas1_folder'], radiosondeSitenum=self.polly_config_dict['radiosondeSitenum'], radiosondeFolder=self.polly_config_dict['radiosondeFolder'], radiosondeType=self.polly_config_dict['radiosondeType'], bgCorrectionIndexLow=self.polly_config_dict['bgCorRangeIndxLow'], bgCorrectionIndexHigh=self.polly_config_dict['bgCorRangeIndxHigh'], bgCorrectionIndexLow_upper=self.polly_config_dict['bgCorRangeIndxLow_upper'], bgCorrectionIndexHigh_upper=self.polly_config_dict['bgCorRangeIndxHigh_upper'], asl=self.polly_config_dict['asl'], initialPolAngle=self.polly_config_dict['init_depAng'], maskPolCalAngle=self.polly_config_dict['maskDepCalAng'], minSNRThresh=self.polly_config_dict['mask_SNRmin'], minPC_fog=self.polly_config_dict['minPC_fog'], flagFarRangeChannel=self.polly_config_dict['isFR'], flag532nmChannel=self.polly_config_dict['is532nm'], flagTotalChannel=self.polly_config_dict['isTot'], flag355nmChannel=self.polly_config_dict['is355nm'], flag607nmChannel=self.polly_config_dict['is607nm'], flag387nmChannel=self.polly_config_dict['is387nm'], flag407nmChannel=self.polly_config_dict['is407nm'], flag355nmRotRaman=np.bitwise_and(np.array(self.polly_config_dict['is355nm']), np.array(self.polly_config_dict['isRR'])).tolist(), # TODO: is this not rdundent as we already should have an RR mask flag532nmRotRaman=np.bitwise_and(np.array(self.polly_config_dict['is532nm']), np.array(self.polly_config_dict['isRR'])).tolist(), # TODO: is this not rdundent as we already should have an RR mask flag1064nmRotRaman=np.bitwise_and(np.array(self.polly_config_dict['is1064nm']), np.array(self.polly_config_dict['isRR'])).tolist(), # TODO: is this not rdundent as we already should have an RR mask. Also this will not work for the 1058nm channel. isUseLatestGDAS=self.polly_config_dict['flagUseLatestGDAS'], collect_debug=collect_debug, flagPicassoComparison=self.polly_config_dict['flagPicassoComparison'], ) self.retrievals_highres.update(preproc_dict)
[docs] def SaturationDetect(self): """Saturation Detection. Attributes ---------- self.flagSaturation : ndarray 1 dimensional temporal boolean array per channel. True if the channel is saturated, else False. """ logging.info("Saturation detection ...") self.flagSaturation = pollySaturationDetect.pollySaturationDetect( data_cube = self, sigSaturateThresh = self.polly_config_dict['saturate_thresh'] )
[docs] def polarizationCaliD90(self, db_path:str=None, collect_debug:bool=False, **defaults): """Calculate/Estimate and select optimal Depol calibration constants with the Delta-90° method. The function calculates/estimates Depol calibration constants (DCs) through the Delta-90° method. If configured, DCs from a defined database are also loaded. The optimal DC per channel is chosen as the DC with the lowest relative error from both the calculated/estimated and loaded DCs. If no DC is found for a channel its default value is taken. Parameters ---------- db_path : str, optional Path to calibration constant database. If ``db_path`` is ``None`` and config variable ``flagUsePreviousDepolCali`` is enabled, a standard path is constructed from the config variable ``calibrationDB``. Default is ``None``. collect_debug : bool, optional If True, collects debug information. Default is False. polCaliEta355 : float, optional Default depol calibration constant at 355 nm. polCaliEtaStd355 : float, optional Default depol calibration constant error at 355 nm. polCaliEta532 : float, optional Default depol calibration constant at 532 nm. polCaliEtaStd532 : float, optional Default depol calibration constant error at 532 nm. polCaliEta1064 : float, optional Default depol calibration constant at 1064 nm. polCaliEtaStd1064 : float, optional Default depol calibration constant error at 1064 nm. Attributes ---------- self.pol_cali['D90' & 'D90_db'] : dict All retrieved and read delta-90° depol calibration constants and retrieval information. self.pol_cali['default'] : dict Default depol calibration constants and retrieval information. self.etaused : dict The optimal depol calibration constants per channel. Notes ----- The function uses the following configuration flags: - ``flagLCCalibration``: Enables or disables calculation of Depol calibration constants through Delta-90° method. - ``flagUsePreviousLC``: Enables or disables loading previously determined Depol calibration constants from the calibration database. All calibration constants are stored in ``self.pol_cali``. The selected optimal constants are stored in ``self.etaused``. Default values are by standard taken from their config variable but can be overwritten if passed as an input to this function. The stuff that starts here in the matlab version https://github.com/PollyNET/Pollynet_Processing_Chain/blob/5efd7d35596c67ef8672f5948e47d1f9d46ab867/lib/interface/picassoProcV3.m#L442 **History** - xxxx-xx-xx: First edition by ... - 2026-10-08: Consider retrieved and database constants equally and added default calibration constants. """ # Load GHK-parameters polarization.loadGHK(self) if self.polly_config_dict['flagDepolCali']: logging.info("Performing Delta-90° Depol calibration ...") # Estimate Depol calibration constants self.pol_cali['D90'] = polarization.calibrateGHK( self, collect_debug=collect_debug ) all_D90_pol_calis = self.pol_cali['D90'].copy() # Check retrieval isUsable = sum([True for val in self.pol_cali['D90'].values() for element in val if element]) if not isUsable: logging.warning("No viable depol calibration constants was retrieved from the measurement.") else: logging.warning("Delta-90° Depol calibration calibration is turned off.") all_D90_pol_calis = {} if self.polly_config_dict['flagUsePreviousDepolCali']: if db_path is None: base_dir = Path(self.picasso_config_dict['results_folder']) db_path = base_dir.joinpath(self.device, self.polly_config_dict['calibrationDB']) logging.info(f"Loading Depol calibration constants from database: {db_path}") # Load DCs from database table_name = 'depol_calibration_constant' ts_interval = self.retrievals_highres['time'][0], self.retrievals_highres['time'][-1] db_table = sql_db.get_from_sql_db( db_path=db_path, table_name=table_name, ts_interval=ts_interval, ts_delta=self.polly_config_dict['TimeInterval4LoadingDBCC'] ) if 'D90_db' in db_table: self.pol_cali['D90_db'] = db_table['D90_db'] # Combine retrieved and database D90 DCs for ch in self.pol_cali['D90_db'].keys(): all_D90_pol_calis[ch] = all_D90_pol_calis.get(ch, []) \ + self.pol_cali['D90_db'][ch] # Load default DCs self.pol_cali['default'] = polarization.loadDefaults(self, **defaults) # Select optimal DCs self.etaused = select.single_best(self.pol_cali['default'], 'eta', 'eta_std', 'method', relative=True) |\ select.single_best(all_D90_pol_calis, 'eta', 'eta_std', 'method', relative=True) # Check if default DCs were used defaultDCsUsed = [ch for ch in self.etaused if self.etaused[ch]['method'] == 'default'] if defaultDCsUsed: logging.warning(f"Deafault Depol calibration constant used for channels: {defaultDCsUsed}")
[docs] def cloudScreen(self, collect_debug:bool=False): """Basic cloud screening. Parameters ---------- collect_debug : bool, optional If true, collects debug information. Default is False. Attributes ---------- self.flagCloudFree : ndarray 1 dimensional temporal boolean array. 0 = cloudy, 1 = cloud free. Notes ----- - Matlab equivalent code at: https://github.com/PollyNET/Pollynet_Processing_Chain/blob/b3b8ec7726b75d9db6287dcba29459587ca34491/lib/interface/picassoProcV3.m#L663 """ logging.info("Cloud screening ...") self.flagCloudFree = cloudscreen.cloudscreen(self, collect_debug=collect_debug)
[docs] def genFlagValPrf(self): """combine the cloudFree, depCal, shutterOn and fog masks to obtain valid profiles """ self.flagValPrf = self.flagCloudFree & (~self.retrievals_highres['depCalMask']) \ & (~self.retrievals_highres['shutterOnMask']) & (~self.retrievals_highres['fogMask'])
[docs] def cloudFreeSeg(self): """Cloud free profile segmentation. Attributes ---------- self.clFreeGrps : nested list List with start, stop index of each cloud free segement. Notes ----- - The matlab eqvivalent code starts here: https://github.com/PollyNET/Pollynet_Processing_Chain/blob/b3b8ec7726b75d9db6287dcba29459587ca34491/lib/interface/picassoProcV3.m#L707 .. TODO:: Write a more extensive example for the docstring. Example ------- .. code-block:: python data_cube.clFreGrps = [ [35, 300], [2500, 2800] ] """ logging.info("Segment cloud free groups ...") self.genFlagValPrf() self.clFreeGrps = profilesegment.segment(self)
[docs] def aggregate_profiles(self, var:str|list=None, func=np.nansum, flagVal:bool=None): """Aggregate highres profiles over cloud free segments. Parameters ---------- var : str or array_like Name(s) of variable to aggregate. Default is None. func : function, optional Function to do the aggregation (mean, sum, median, etc.). Default is np.nansum. flagVal : None | bool | np.ndarray cut out single invalid profiles form `clFreeGrps`. If `True` the `data_cube.flagValPrf` is used. Individual array can also be supplied. Attributes ---------- self.retrievals_profiles[`var`] : ndarray Aggregated profile. Notes ----- - The variable(s) `var` need to be stored in the dictionary data_cube.retrievals_highres. All aggregated profiles will be stored in the dictionary data_cube.retrievals_profiles under the the same key. - If var is None. The following variables will be aggregated: `sigBGCor`, `BG`, `RCS`, `mShots`, `mask387Off`, `mask607Off`, `mask407Off` - `func=np.nansum` is needed for correct SNR calculations. ** History ** - xxxx-xx-xx: First edition by ... - 2026-07-28: Added option to aggregate multiple variables in the same call. And corrected the aggregation for PCR-signals like `RCS`. """ if isinstance(flagVal, (np.ndarray, list)): flagVal = np.asarray(flagVal) self.flagValPrf = flagVal elif flagVal and not isinstance(flagVal, (np.ndarray, list)): flagVal = self.flagValPrf if var is None: # Take care of the default scenario self.aggregate_profiles(['sigBGCor', 'BG', 'RCS', 'mShots'], flagVal=flagVal) self.aggregate_profiles(['mask387Off', 'mask607Off', 'mask407Off'], func=np.nanmean, flagVal=flagVal) return if isinstance(var, str): var = [var] for variable in var: logging.info(f"Aggregating variable: {variable} ...") if variable in self.retrievals_highres: if variable == "RCS": if self.polly_config_dict['flagPicassoComparison']: self.retrievals_profile[variable] = \ preprocprofiles.aggregate_clFreeGrps(self, variable, func=np.nanmean, flagVal=flagVal) else: self.retrievals_profile[variable] = pollyPreprocess.calculate_rcs( signal=pollyPreprocess.photonCount2PCR( signal=preprocprofiles.aggregate_clFreeGrps(self, 'sigBGCor', func=func, flagVal=flagVal), mShots=preprocprofiles.aggregate_clFreeGrps(self, 'mShots', func=func, flagVal=flagVal), hRes=self.rawdata_dict['measurement_height_resolution']['var_data'] ), ranges=self.retrievals_highres['range'] ) else: self.retrievals_profile[variable] = \ preprocprofiles.aggregate_clFreeGrps(self, variable, func=func, flagVal=flagVal) else: logging.critical(f"{variable} is NOT in data_cube.retrievals_highres") raise ValueError(f"Could not locate variable '{variable}'.")
[docs] def loadMeteo(self): """Load meteorological data. Meteorological data like; - Temperatue - Pressure - Relative Humidity - Spesific Humidity is read form the cloudnet ECMWF file specified by the config variable; - `meteorDataSource` - `meteo_folder` - `meteo_file` and saved in the dataFrame `self.met`. Attributes ---------- self.met : object Object to handle meterological data. Notes ----- - Only `meteorDataSource = 'nc_cloudnet'` is currently suported. """ logging.info("Loading meteorological data ...") self.met = readMeteo.Meteo( self.polly_config_dict['meteorDataSource'], self.polly_config_dict['meteo_folder'], self.polly_config_dict['meteo_file'] ) self.met.load( times=datetime.datetime.timestamp(datetime.datetime.strptime(self.date, '%Y%m%d')), heights=self.retrievals_highres['height'], asl=float(self.polly_config_dict['asl']), flagPicassoComparison=self.polly_config_dict['flagPicassoComparison'] )
[docs] def loadAOD(self): """Load the AOD from a co-located fotometer. Notes ----- .. TODO:: Implement the option to load AOD from co-located fotometer. """ logging.critical('Function loadAOD is not yet implemented!') raise NotImplementedError('Function loadAOD is not yet implemented!')
[docs] def calcMolecular(self): """Calculate the molecular scattering for the cloud free periods with the strategy of first averaging the met data and then calculating the rayleigh scattering. Attributes ---------- self.mol_profiles : dict Dictionary containing calculated molecular profiles for each channel. Keys ---- mBsc_355 : ndarray (channels, height) Molecular backscatter at 355 nm [m^{-1}Sr^{-1}]. mExt_355 : ndarray (channels, height) Molecular extinction at 355 nm [m^{-1}]. mBsc_387 : ndarray (channels, height) Molecular backscatter at 387 nm [m^{-1}Sr^{-1}]. mExt_387 : ndarray (channels, height) Molecular extinction at 387 nm [m^{-1}]. mBsc_407 : ndarray (channels, height) Molecular backscatter at 407 nm [m^{-1}Sr^{-1}]. mExt_407 : ndarray (channels, height) Molecular extinction at 407 nm [m^{-1}]. mBsc_532 : ndarray (channels, height) Molecular backscatter at 532 nm [m^{-1}Sr^{-1}]. mExt_532 : ndarray (channels, height) Molecular extinction at 532 nm [m^{-1}]. mBsc_607 : ndarray (channels, height) Molecular backscatter at 607 nm [m^{-1}Sr^{-1}]. mExt_607 : ndarray (channels, height) Molecular extinction at 607 nm [m^{-1}]. mBsc_1058 : ndarray (channels, height) Molecular backscatter at 1058 nm [m^{-1}Sr^{-1}]. mExt_1058 : ndarray (channels, height) Molecular extinction at 1058 nm [m^{-1}]. mBsc_1064 : ndarray (channels, height) Molecular backscatter at 1064 nm [m^{-1}Sr^{-1}]. mExt_1064 : ndarray (channels, height) Molecular extinction at 1064 nm [m^{-1}]. number_density : ndarray (channels, height) Number density. Notes ----- .. TODO:: How are these molecular profiles aggregated (mean or sum??). .. TODO:: Idea: As we already need the highres molecular signal for the WV-product. Would it not make more sens to retrieve this first and then use the aggregate_profile() function to get the cloud free profiles? --> This only makes sens if we can directly retrieve the highres molecular signal which I dout we can. """ logging.info("Calculating molecular profiles ...") time_slices = [self.retrievals_highres['time64'][grp] for grp in self.clFreeGrps] logging.info(f"time slices of cloud free {time_slices}") mean_profiles = self.met.get_mean_profiles(time_slices) self.mol_profiles = molecular.calc_profiles(mean_profiles, flagPicassoComparison=self.polly_config_dict['flagPicassoComparison'])
[docs] def watervaporCali(self, db_path:str=None, collect_debug:bool=False, **defaults): """Perform water vapor calibration on aggregated profiles. Parameters ---------- db_path : str, optional Path to calibration constant database. If ``db_path`` is ``None`` and config variable ``flagUsePreviousWVconst`` is enabled, a standard path is constructed from the config variable ``calibrationDB``. Default is ``None``. collect_debug : bool, optional If True, collects debug information. Default is False. wvconst : float Default water vapor calibration constant. wvconstStd : float Default water vapor calibration constant error. Attributes ---------- self.wv_cali['model_profile' & 'model_profile_db'] : dict All retrieved and read water vapor calibration constants and retrieval information from model data with profile method. self.wv_cali['model_regression' & 'model_regression_db'] : dict All retrieved and read water vapor calibration constants and retrieval information from model data with regression method. self.wv_cali['default'] : dict Default depol calibration constants and retrieval information. self.WVCused : dict The optimal depol calibration constants. Notes ----- The function uses the following configuration flags: - ``flagWVCalibration``: Enables or disables calculation of water vapor calibration constants through Delta-90° method. - ``flagUsePreviousWVconst``: Enables or disables loading previously determined water vapor calibration constants from the calibration database. All calibration constants are stored in ``self.wv_cali``. The selected optimal constants are stored in ``self.WVCused``. Default values are by standard taken from their config variable but can be overwritten if passed as an input to this function. At the moment, only calibration results form the model regression method is considerd when selecting the optimal constant. There currently exist no option to select the best WVC over all calibration instruments and methods. .. TODO:: How to select best WVC over all calibration instruments and methods. **History** - xxxx-xx-xx: First edition by ... - 2026-09-11: Implemented in python by Jakob. """ if self.polly_config_dict['flagWVCalibration']: logging.info('Performing Water Vapor calibration ...') # Estimate WV calibration constants self.wv_cali['model_profile'], self.wv_cali['model_regression'] = watervapor.wvc_for_cldFreeGrps( self, instrument='model', collect_debug=collect_debug ) all_modelProfile_WVCs = self.wv_cali['model_profile'].copy() all_modelRegression_WVCs = self.wv_cali['model_regression'].copy() # Check retrieval isUsable = sum([True for val in list(self.wv_cali['model_profile'].values()) \ + list(self.wv_cali['model_regression'].values()) for element in val if element]) if not isUsable: logging.warning("No viable water vapor calibration constants was retrieved from the measurement.") else: logging.warning("Water Vapor calibration is turned off.") all_modelProfile_WVCs, all_modelRegression_WVCs = {}, {} if self.polly_config_dict['flagUsePreviousWVconst']: if db_path is None: base_dir = Path(self.picasso_config_dict['results_folder']) db_path = base_dir.joinpath(self.device, self.polly_config_dict['calibrationDB']) logging.info(f"Loading Water Vapor calibration constants from database: {db_path}") # Load WVCs from database table_name = 'wv_calibration_constant' ts_interval = self.retrievals_highres['time'][0], self.retrievals_highres['time'][-1] db_table = sql_db.get_from_sql_db( db_path=db_path, table_name=table_name, ts_interval=ts_interval, ts_delta=self.polly_config_dict['TimeInterval4LoadingDBCC'] ) if 'model_profile_db' in db_table: self.wv_cali['model_profile_db'] = db_table['model_profile_db'] # Combine retrieved and database model profile WVCs for ch in self.wv_cali['model_profile_db']: all_modelProfile_WVCs[ch] = all_modelProfile_WVCs.get(ch, []) \ + self.wv_cali['model_profile_db'][ch] if 'model_regression_db' in db_table: self.wv_cali['model_regression_db'] = db_table['model_regression_db'] # Combine retrieved and database model regression WVCs for ch in self.wv_cali['model_regression_db']: all_modelRegression_WVCs[ch] = all_modelRegression_WVCs.get(ch, []) \ + self.wv_cali['model_regression_db'][ch] # Load default WVCs self.wv_cali['default'] = watervapor.loadDefaults(self, **defaults) # Select optimal WVCs self.WVCused = select.single_best(self.wv_cali['default'], 'WVC', 'WVCStd', 'method', relative=True) |\ select.single_best(all_modelRegression_WVCs, 'WVC', 'WVCStd', 'method', relative=True) # Check if default WVCs were used defaultWVCsUsed = [ch for ch in self.WVCused if self.WVCused[ch]['method'] == 'default'] if defaultWVCsUsed: logging.warning(f"Deafault Water vapor calibration constant used for channels: {defaultWVCsUsed}")
[docs] def rayleighFit(self, collect_debug:bool=False): """Perform the rayleigh fit procedure. Parameters ---------- collect_debug : bool, optional If true, collects debug information. Default is False. Attributes ---------- self.retrievals_profiles['refH'][idx][ch] : dict Reference height information for cloud free segment `idx` and channel `ch`. Keys ---- DPInd : list List of Douglas-Peckur indecies used as potensial reference index candidates. refInd : list Indices (start, stop) of the retrieved reference height. refHeigt : list Height above ground (start, stop) of the retrieved reference height. refRange : list Height above ground at zenith angle (start, stop) of the retrieved reference height. Notes ----- - Direct translation from the matlab code. There might be noticeable numerical discrepancies (especially in the residual) seemed to work ok for 532, 1064, but with issues for 355. - The Douglas Peucker algorithm works fine (gives the same result as the Matlab version). However, due to the numerical discrepancies In the residuals of the fit algorithm sometimes different refH segments are chosen. """ logging.info('Start Rayleigh Fit') logging.warning(f'Potential for differences to matlab code due to numerical issues (subtraction of two small values)') self.retrievals_profile['refH'] = rayleighfit.rayleighfit(self, collect_debug)
[docs] def polarizationCaliMol(self): """Calibration with molecular signal in reference height. Attributes ---------- self.pol_cali['mol'][ch][idx] : dict Molecular depol calibration constants and retrieval information for channel `ch` and cloud free index `idx`. keys ---- eta : ndarray Polarization calibration eta. etaStd : ndarray Uncertainty of polarization calibration eta. fac : array Polarization calibration factor. facStd : ndarray Uncertainty of polarization calibration factor. status : int Retrieval status 0 : Bad 1 : Good time_start : int Start time of the cloud free segment for the retrieval in unixtime. end_time : int End time of the cloud free segment for the retrieval in unixtime. Notes ----- - Calibration is only performed if the config variable `flagMolDepolCali = True`. """ if self.polly_config_dict['flagMolDepolCali']: logging.info("Calibrating molecular signal ...") logging.warning(f'not checked against the matlab code') self.pol_cali['mol'] = polarization.calibrateMol(self) else: logging.warning("'flagMolDepolCali' set to False")
[docs] def transCor(self): """Perform GHK-transmission correction on the signal. Attributes ---------- self.retrievals_highres : dict With corrected signal. Keys ---- sigTCor : ndarray GHK-Transmission corrected signal [MCPS]. BGTCor : ndarray GHK-Transmission corrected background. Notes ----- - if the config entry `flagTransCor = False`, no GHK transmission correction will be done and the background corrected signal will replace the transmission corrected signal for the rest of the processing steps. .. TODO:: Overwriting `sigTCor` with `sigBGCor` when `flagTransCor = False` can cause confiusion among useres. It would be more intutive to not have a `sigTCor` variable in this case. However, this requiers re... the rest of the processing chain. """ if self.polly_config_dict['flagTransCor']: logging.info('GHK Transmission correction ...') self.retrievals_highres['sigTCor'], self.retrievals_highres['BGTCor'] = \ transCor.transCorGHK_cube(self) else: logging.warning('No GHK transmission correction done.') self.retrievals_highres['sigTCor'], self.retrievals_highres['BGTCor'] = \ self.retrievals_highres['sigBGCor'], self.retrievals_highres['BG']
[docs] def retrievalKlett(self, oc:bool=False, nr:bool=False): """Apply Klett retrieval. Parameters ---------- oc : bool, optional If true, apply the retrieval on the overlap corrected profiles. The outupt will then also be saved under the key 'klett_OC' insted of 'klett'. Default is False. nr : bool, optional If true, apply the retrieval on the near range profiles as well as the far range profiles. Default is False. Attributes ---------- self.retrievals_profile['klett' or 'klett_OC'][idx][ch] : dict Klett retrieved optical profiles and retrieval information for cloud free segment `idx` and channel `ch`. Keys ---- aerExt : ndarray Aerosol extinction coefficient [m^{-1}]. aerExtStd : ndarray Uncertainty of aerosol extinction coefficient [m^{-1}]. aerBsc : ndarray Aerosol backscatter coefficient [m^{-1}Sr^{-1}]. aerBscStd : ndarray Uncertainty of aerosol backscatter coefficient [m^{-1}Sr^{-1}]. aerBR : ndarray Aerosol backscatter ratio. aerBRStd : ndarray Statistical uncertainty of aerosol backscatter ratio. retrieval : str Name of retrieval type, eg. 'klett'. signal : str Name of the signal used for the retrieval, eg. 'TCor'. refBeta : float Reference value used for the retrieval [m^{-1}Sr^{-1}]. Notes ----- - The retrieval is by default applied on the FR profiles. if `nr=True` the retrieval will be applied both on the NR and FR profiles. """ retrievalname = 'klett' kwargs = {} if oc: # Check if overlap corrected signal exist if "sigOLCor" not in self.retrievals_highres: # This should be done more general not only for OLCor but for other signals as well. logging.warning("No overlap corrected signal found.") return retrievalname +='_OC' kwargs['signal'] = 'OLCor' if nr: kwargs['nr'] = True logging.info(f"Klett retrieval for FR {'& NR' if nr else None} {'overlap' if oc else 'GHK-transmission'} corrected signal ...") self.retrievals_profile[retrievalname] = klettfernald.run_cldFreeGrps(self, **kwargs) if retrievalname not in self.retrievals_profile['avail_optical_profiles']: self.retrievals_profile['avail_optical_profiles'].append(retrievalname)
[docs] def retrievalRaman(self, oc:bool=False, nr:bool=False, collect_debug:bool=False): """Apply raman retrieval (nighttime only). Parameters ---------- oc : bool, optional If true, apply the retrieval on the overlap corrected profiles. The outupt will then also be saved under the key 'raman_OC' insted of 'raman'. Default is False. nr : bool, optional If true, apply the retrieval on the near range profiles as well as the far range profiles. Default is False. collect_debug : bool, optional If true, collects debug information. Default is False. Attributes ---------- self.retrievals_profile['raman' or 'raman_OC'][idx][ch] : dict Raman retrieved optical profiles and retrieval information for cloud free segment `idx` and channel `ch`. Keys ---- aerExt : ndarray Aerosol extinction coefficient [m^{-1}]. aerExtStd : ndarray Uncertainty of aerosol extinction coefficient [m^{-1}]. aerBsc : ndarray Aerosol backscatter coefficient [m^{-1}Sr^{-1}]. aerBscStd : ndarray Uncertainty of aerosol backscatter coefficient [m^{-1}Sr^{-1}]. LR : ndarray Aerosol Lidar ratio [sr]. effRes : ndarray Effective resolution of aerosol lidar ratio [m]. LRStd : ndarray Uncertainty of aerosol lidar ratio [sr]. retrieval : str Name of retrieval type eg. 'raman'. signal : str Name of the signal used for the retrieval eg. 'TCor'. refBeta : float Reference value used for the retrieval. Notes ----- - The retrieval is by default applied on the FR profiles. if `nr=True` the retrieval will be applied both on the NR and FR profiles. """ retrievalname = 'raman' kwargs = {} if oc: # Check if overlap corrected signal exist if "sigOLCor" not in self.retrievals_highres: # This should be done more general not only for OLCor but for other signals as well. logging.warning("No overlap corrected signal found.") return retrievalname +='_OC' kwargs['signal'] = 'OLCor' # get the full overlap height for the overlap corrected variant # group by the cloud free groups kwargs['heightFullOverlap'] = \ [np.mean(self.retrievals_highres['heightFullOverCor'][slice(*cF)], axis=0) for cF in self.clFreeGrps] if nr: kwargs['nr'] = True kwargs['collect_debug'] = collect_debug logging.info(f"Raman retrieval for FR {'& NR' if nr else None} {'overlap' if oc else 'GHK-transmission'} corrected signal ...") self.retrievals_profile[retrievalname] = raman.run_cldFreeGrps(self, **kwargs) if retrievalname not in self.retrievals_profile['avail_optical_profiles']: self.retrievals_profile['avail_optical_profiles'].append(retrievalname)
[docs] def overlapCalc(self, collect_debug:bool=False): """Estimate the overlap function. Parameters ---------- collect_debug : str, optional If true, collects debug information. Default is False. Attributes ---------- self.retrievals_profile['overlap']['frnr'][idx][ch] : list of dicts Far-range-Near-range retrieved overlap function and retrieval information for cloud free segment `idx` and channel `ch`. Keys ---- olFunc : ndarray Overlap function. olFuncStd : ndarray Standard deviation of overlap function. sigRatio : float Signal ratio between near-range and far-range signals. normRange : list Height index of the signal normalization range. self.retrievals_profile['overlap']['raman'][idx][ch] : list of dicts Raman retrieved overlap function and retrieval information for cloud free segment `idx` and channel `ch`. Keys ---- olFunc : ndarray Overlap function. olFunc_raw : ndarray Overlap function with no smoothing. LR : float Optimal Lidar Ratio for ... normRange : list Height index of the signal normalization range. Notes ----- - Different to the matlab version, where an average over all cloud free periods is taken. Here the overlap is applied per cloud free period. """ if not self.polly_config_dict["flagOLCor"]: return logging.info("calculating overlap functions ...") self.retrievals_profile['overlap'] = {} self.retrievals_profile['overlap']['frnr'] = overlapEst.run_frnr_cldFreeGrps(self, collect_debug=collect_debug) self.retrievals_profile['overlap']['raman'] = overlapEst.run_raman_cldFreeGrps(self, collect_debug=collect_debug)
[docs] def overlapFixLowestBins(self): """The lowest bins are affected by stange near range effects. Attributes ---------- self.retrievals_profile['overlap']['frnr'][idx][ch]['olFunc'] : ndarray FRNR retrieved overlap func with subdued near-range effect. self.retrievals_profile['overlap']['raman'][idx][ch]['olFunc'] : ndarray Raman retrieved overlap func with subdued near-range effect. """ if not self.polly_config_dict["flagOLCor"]: return height = self.retrievals_highres['range'] for k in self.retrievals_profile['overlap']: logging.info(f"Fixing lower bins for {k} overlap functions") overlapCor.fixLowest(self.retrievals_profile['overlap'][k], np.where(height > 800)[0][0])
[docs] def overlapCor(self): """Overlap correction. Attributes ---------- self.retrieval_highres : dict With overlap corrected signal and retrieval information. Keys ---- overlap2d : ndarray 2 dimensional (time, height) overlap function. sigOLCor : ndarray Overlap corrected signal [photon count]. BGOLCor : ndarray Overlap corrected background. heightFullOverCor : ndarray Retrieved height of full overlap [m]. Notes ----- - the overlap correction is implemented differently to the matlab version first a 2d (time, height) correction array is constructed then it is applied. In future this will allow for time varing overlap functions """ if self.polly_config_dict['overlapCorMode'] == 0 or not self.polly_config_dict["flagOLCor"]: logging.info('no overlap correction applied') return logging.info('Apply overlap correction ...') if self.polly_config_dict['overlapCorMode'] == 1: logging.info('overlapCorMode 1 -> need file for overlapfunction') if not 'overlap' in self.retrievals_profile: self.retrievals_profile['overlap'] = {} self.retrievals_profile['overlap']['file'] = overlapEst.load(self) self.retrievals_highres['overlap2d'] = overlapCor.spread(self) ret = overlapCor.apply_cube(self) self.retrievals_highres['sigOLCor'] = ret[0] self.retrievals_highres['BGOLCor'] = ret[1] self.retrievals_highres['heightFullOverCor'] = ret[2]
[docs] def calcDepol(self): """Calculate the volume depol and the particle depol. Attributes ---------- self.retrievals_profiles[sig][idx][ch] : dict Volume and particle depolarization profiles and retrieval info for signal type `sig` for cloud free segment `idx` and channel `ch`. Keys ---- vdr : ndarray Volume depolarization ratio []. vdrStd : ndarray Uncertainty of the volume depolarization ratio. mdr : ndarray Molecular depolarization ratio []. mdrStd : ndarray Uncertainty of the molecular depolarization ratio. pdr : ndarray Particle depolarization ratio []. pdrStd : ndarray Uncertainty of the particle depolarization ratio. Notes ----- - This retrieval is done for all avaiable optical profiles. """ for ret_prof_name in self.retrievals_profile['avail_optical_profiles']: logging.info(f"Calculate volume and particle depolarization ratios for product {ret_prof_name} ...") self.retrievals_profile[ret_prof_name] = depolarization.voldepol_cldFreeGrps( self, ret_prof_name) self.retrievals_profile[ret_prof_name] = depolarization.pardepol_cldFreeGrps( self, ret_prof_name)
[docs] def estQualityMask(self): """Estimate the quality mask. Attributes ---------- self.retrievals_highres['quality_mask'] : ndarray High resolution (time, height) quality masks per channel. 0 : good data 1 : low-SNR data 2 : depolarization calibration periods 3 : shutter on 4 : fog 5 : saturated (NEW) Notes ----- - If the config variables `flagUseImprovedSNR=True`, SNR and the lowSNRMask are recalculated with quasi-smoothed signals to improve data quality. ** History ** - xxxx-xx-xx: First edition by ... - 2026-07-17: Added quasi-smoothed SNR and lowSNRMask. """ logging.info("Estimate quality masks ...") if self.polly_config_dict['flagUseImprovedSNR']: self.retrievals_highres['SNR_quasi'], self.retrievals_highres['lowSNRMask_quasi'] = qualityMask.improvedSNR(self) self.retrievals_highres['quality_mask'] = qualityMask.qualityMask(self)
[docs] def Angstroem(self): """Calculate the angstrom exponent. Attributes ---------- self.retrievals_profiles[sig][idx][ch] : dict Volume and particle depolarization profiles and retrieval info for signal type `sig` for cloud free segment `idx` and channel `ch`. Keys ---- AE_{prod}_{wv1}_{wv2} : ndarray Angstroem Exponent for product `prod`, and wavelengths `wv1` and `wv2`. AEStd_{prod}_{wv1}_{wv2} : ndarray Uncertainty of Angstroem Exponent for product `prod`, and wavelengths `wv1` and `wv2`. Notes ----- - AE_{prod}_{wv1}_{wv2} = log(prod_wv1 / prod_wv2) / log(wv2 / wv1) - This retrieval is done for all avaiable optical profiles. """ for ret_prof_name in self.retrievals_profile['avail_optical_profiles']: logging.info(f"Calculate Angstrom exponents for product {ret_prof_name} ...") self.retrievals_profile[ret_prof_name] = angstroem.ae_cldFreeGrps( self, ret_prof_name)
[docs] def LidarCalibration(self, db_path:str=None, collect_debug:bool=False, **defaults): """Calculate/Estimate and select optimal lidar calibration constants. The function calculates/estimates lidar calibration constants (LCs) from both klett and Raman retrieved optical profiles. If configured, LCs from a defined database are also loaded. The optimal LC per channel is chosen as the LC with the lowest relative error per channel form both the calculated/estimated and loaded LCs. If no LC is found for a channel its default value is taken. Parameters ---------- db_path : str or Path, optional Path to calibration constant database. If ``db_path`` is ``None`` and config variable ``flagUsePreviousLC`` is enabled, a standard path is constructed from the config variable ``calibrationDB``. Default is ``None``. collect_debug : bool, optional If ``True``, collects debug information. Default is ``False``. LC : list, optional Default Lidar constant value per channel. LCStd : list, optional Default Lidar constant error per channel. Attributes ---------- self.LC['klett' & 'klett_db'] : dict All retrieved and read Klett lidar calibration constants and retrieval information. self.LC['raman' & 'raman_db'] : dict All retrieved and read Raman lidar calibration constants and retrieval information. self.LC['default'] : dict Default lidar calibration constants and retrieval information. self.LCused : dict The optimal lidar calibration constant per channel. Notes ----- The function uses the following configuration flags: - ``flagLCCalibration``: Enables or disables calculation of Lidar calibration constants from klett and Raman retrieval. - ``flagUsePreviousLC``: Enables or disables loading previously determined lidar calibration constants from the calibration database. All calibration constants are stored in ``self.LC``. The selected optimal constants are stored in ``self.LCused``. Default values are by standard taken from their config variable but can be overwritten if passed as an input to this function. **History** - xxxx-xx-xx: First edition by ... - 2026-10-08: Consider retrieved and database constants equally and added default calibration constants. """ if self.polly_config_dict['flagLCCalibration']: logging.info("Performing Lidar calibration ...") # Estimate Lidar constant from klett retrieval self.LC['klett'] = lidarconstant.lc_for_cldFreeGrps( self, retrieval='klett', collect_debug=collect_debug ) all_klett_LCs = self.LC['klett'].copy() # Estimate Lidar constant from raman retrieval self.LC['raman'] = lidarconstant.lc_for_cldFreeGrps( self, retrieval='raman', collect_debug=collect_debug ) all_raman_LCs = self.LC['raman'].copy() # Check retrieval isUsable = sum([True for val in list(self.LC['klett'].values()) \ + list(self.LC['raman'].values()) for element in val if element]) if not isUsable: logging.warning("No viable Lidar calibration constants was retrieved from the measurement.") else: logging.warning("Lidar calibration is turned off.") all_klett_LCs, all_raman_LCs = {}, {} if self.polly_config_dict['flagUsePreviousLC']: if db_path is None: base_dir = Path(self.picasso_config_dict['results_folder']) db_path = base_dir.joinpath(self.device, self.polly_config_dict['calibrationDB']) logging.info(f"Loading Lidar calibration constants from database: {db_path}") # Load LCs from database table_name = 'lidar_calibration_constant' ts_interval = self.retrievals_highres['time'][0], self.retrievals_highres['time'][-1] db_table = sql_db.get_from_sql_db( db_path=db_path, table_name=table_name, ts_interval=ts_interval, ts_delta=self.polly_config_dict['TimeInterval4LoadingDBCC'] ) if 'klett_db' in db_table: self.LC['klett_db'] = db_table['klett_db'] # Combine retrieved and database Klett LCs for ch in self.LC['klett_db']: all_klett_LCs[ch] = all_klett_LCs.get(ch, []) \ + self.LC['klett_db'][ch] if 'raman_db' in db_table: self.LC['raman_db'] = db_table['raman_db'] # Combine retrieved and database Raman LCs for ch in self.LC['raman_db']: all_raman_LCs[ch] = all_raman_LCs.get(ch, []) \ + self.LC['raman_db'][ch] # Load default LCs self.LC['default'] = lidarconstant.loadDefaults(self, **defaults) # Select optimal LCs self.LCused = select.single_best(self.LC['default'], 'LC', 'LCStd', 'method', relative=True) |\ select.single_best(all_klett_LCs, 'LC', 'LCStd', 'method', relative=True) |\ select.single_best(all_raman_LCs, 'LC', 'LCStd', 'method', relative=True) # Check if default LCs were used defaultLCsUsed = [ch for ch in self.LCused if self.LCused[ch]['method'] == 'default'] if defaultLCsUsed: logging.warning(f"Deafault Lidar calibration constant used for channels: {defaultLCsUsed}")
[docs] def attBsc_volDepol(self): """High resolution attenuated backscatter and volume depolarization. Attributes ---------- self.retrievals_highres : dict With attenuated backscatter. Keys ---- attBsc_{wv}\_{t}\_{tel} : ndarray High resolution (time, height) attenuated backscatter at channel {wv}\_{t}\_{tel}. attBsc_{wv}\_{t}\_OC : ndarray High resolution (time, height) overlap corrected attenuated backscatter at channel {wv}\_{t}\_FR. """ # for now try with mutable state in data_cube logging.info("2D attenuated backscatter retrieval ...") highres.attbsc_2d(self) logging.info("2D volume depolarization ratio retrieval ...") highres.voldepol_2d(self)
[docs] def molecularHighres(self): """Calculate the molecular signal for the 2d high resolution. Attributes ---------- self.mol_2d : xr.Dataset xarray dataset with dimensions time and height and variables molecular backscatter (mBsc) and molecular extinction (mExt) per wavelength. Notes ----- - mol_2d's time dimension differe from the time dimension of the measurement. """ logging.info("2D molecular retrieval ...") self.mol_2d = molecular.calc_2d( self.met.ds, flagPicassoComparison=self.polly_config_dict['flagPicassoComparison'] )
[docs] def wvmr(self): """High resolution water vapor mixing ratio. Attributes ---------- self.retrieval_highres : dict With water vapor mixing rato. Keys ---- wvmr_{wv}_{tel} : ndarray High resolution (time, height) water vapor mixing ratio fro channel {wv}\_{tel}. """ logging.info("2D water vapor mixing ratio retrieval ...") highres.wvmr_2d(self)
[docs] def quasiV1(self): """QuasiV1 retrievals and target categorisation. Attributes ---------- self.retrieval_highres : dict With high resolution Quasi version 1 products as well as Target categorization. Keys ---- quasiBscV1_{wv}\_{t}\_{tel} : ndarray Quasi version 1 particle backscatter at channel `{wv}_{t}_{tel}`. quasiExtV1_{wv}\_{t}\_{tel} : ndarray Quasi version 1 particle extinction at channel `{wv}_{t}_{tel}`. quasiVdrV1_{wv}\_{t}\_{tel} : ndarray Quasi version 1 volume depolarization ratio at channel `{wv}_{t}_{tel}`. (usally `wv =532`). quasiPdrV1_{wv}\_{t}\_{tel} : ndarray Quasi version 1 particle depolarization ratio at channel `{wv}_{t}_{tel}`. (usally `wv =532`). quasiAEV1_{wv1}\_{wv2} : ndarray Quasi version 1 Angstroem exponent between two wavelengets `wv1` and `wv2`. tcMaskV1 : ndarray Classification mask version 1 (time x height). 0: No signal 1: Clean atmosphere 2: Non-typed particles/low conc. 3: Aerosol: small 4: Aerosol: large, spherical 5: Aerosol: mixture, partly non-spherical 6: Aerosol: large, non-spherical 7: Cloud: non-typed 8: Cloud: water droplets 9: Cloud: likely water droplets 10: Cloud: ice crystals 11: Cloud: likely ice crystal Notes ----- - QuasiV1 products uses Klett-retrieved products. """ logging.info('Calculating Quasi V1 particle backscatter coefficient') quasiV1.quasi_bsc(self) logging.info('Calculating Quasi V1 particle Depolarization ratio') quasi.quasi_pdr(self, version='V1') logging.info('Calculating Quasi V1 Ångström Exponent') quasi.quasi_angstrom(self, version='V1') logging.info('Producing V1 target categorization') quasi.target_cat(self, version='V1')
[docs] def quasiV2(self): """QuasiV2 retrievals and target categorisation. Attributes ---------- self.retrieval_highres : dict With high resolution Quasi version 2 products as well as Target categorization. Keys ---- quasiBscV2_{wv}\_{t}\_{tel} : ndarray Quasi version 2 particle backscatter at channel `{wv}_{t}_{tel}`. quasiExtV2_{wv}\_{t}\_{tel} : ndarray Quasi version 2 particle extinction at channel `{wv}_{t}_{tel}`. quasiVdrV2_{wv}\_{t}\_{tel} : ndarray Quasi version 2 volume depolarization ratio at channel `{wv}_{t}_{tel}`. (usally `wv =532`). quasiPdrV2_{wv}\_{t}\_{tel} : ndarray Quasi version 2 particle depolarization ratio at channel `{wv}_{t}_{tel}`. (usally `wv =532`). quasiAEV2_{wv1}\_{wv2} : ndarray Quasi version 2 Angstroem exponent between two wavelengets `wv1` and `wv2`. tcMaskV2 : ndarray Classification mask version 2 (time x height). 0: No signal 1: Clean atmosphere 2: Non-typed particles/low conc. 3: Aerosol: small 4: Aerosol: large, spherical 5: Aerosol: mixture, partly non-spherical 6: Aerosol: large, non-spherical 7: Cloud: non-typed 8: Cloud: water droplets 9: Cloud: likely water droplets 10: Cloud: ice crystals 11: Cloud: likely ice crystal Notes ----- - QuasiV2 products uses Raman-retrieved products. """ logging.info('Calculating Quasi V2 particle backscatter coefficient') quasiV2.quasi_bsc(self) logging.info('Calculating Quasi V2 particle Depolarization ratio') quasi.quasi_pdr(self, version='V2') logging.info('Calculating Quasi V2 Ångström Exponent') quasi.quasi_angstrom(self, version='V2') logging.info('Producing V2 target categorization') quasi.target_cat(self, version='V2')
[docs] def write_2_sql_db(self, parameter:str, db_path:str|None=None, method:str|None=None): """Write LC, eta or WVC to sqlite db table. Parameters ---------- parameter : str can be LC (Lidar-calibration-constant), DC (Depol-calibration-constant) or WVC (WaterVapor-calibration-constant) method : str 'raman', 'klett', 'D90', 'transfer', 'profile' or 'regression' db_path : str location of the sqlite db-file Notes ----- - The unique columns are needed that new entries overwrite old ones, otherwise they are just added to the table with same timestamps. """ if db_path == None: base_dir = Path(self.picasso_config_dict['results_folder']) db_path = base_dir.joinpath(self.device, self.polly_config_dict['calibrationDB']) logging.info(f"read db_path from polly_config_dict {db_path}") table_name = sql_db.DB_SCHEME[parameter]['table_name'] column_names = sql_db.DB_SCHEME[parameter]['column_names'] data_types = sql_db.DB_SCHEME[parameter]['data_types'] unique = sql_db.DB_SCHEME[parameter]['unique'] assert len(column_names) == len(data_types), 'column names do not match data types' logging.info(f'writing to sqlite-db: {db_path}') logging.info(f'writing {parameter} to table: {table_name}') sql_db.setup_empty(db_path, table_name, column_names, data_types, unique=unique) rows_to_insert = sql_db.prepare_for_sql_db_writing(self, parameter, method) sql_db.write_rows_to_sql_db(db_path, table_name, column_names, rows_to_insert)
[docs] def read_calibration_db(self, db_path:str|None=None): """Read the calibration constants from database. Parameters ---------- db_path : str or NoneType path to database file... Default is None. Attributes ---------- self.LC : dict Original `LC` with all additinal lidar calibration constant availabel in the datafram `db_path` in the time period of the measurement +-24h. self.pol_cali : dict Original `pol_cali` with all additinal polarization calibration constant availabel in the datafram `db_path` in the time period of the measurement +-24h. Notes ----- - Time interval includes 24h before start of the measurement and after the end of the measurement. """ if db_path == None: db_path = self.polly_config_dict['calibrationDB'] logging.info(f"read db_path from polly_config_dict {db_path}") ts_interval = self.retrievals_highres['time'][0], self.retrievals_highres['time'][-1] table_name = 'lidar_calibration_constant' self.LC.update(sql_db.get_from_sql_db(db_path, table_name, ts_interval)) table_name = 'depol_calibration_constant' self.pol_cali.update(sql_db.get_from_sql_db(db_path, table_name, ts_interval)) table_name = 'wv_calibration_constant' self.wv_cali.update(sql_db.get_from_sql_db(db_path, table_name, ts_interval))
[docs] def adding_retrieving_infos_2_polly_config_dict(self): """Some infos from the polly_config_dict should have there own keys, e.g. reference_search_range. Attributes ---------- self.polly_config_dict : dict With additional info. Keys ---- reference_search_range_355_total_FR : list ... reference_search_range_532_total_FR : list ... reference_search_range_1064_total_FR : list ... Notes ----- .. TODO :: - Finish docstring. - Is this function necessary? """ lower_overlap = np.array(self.polly_config_dict['heightFullOverlap']) reference_search_range_355_total_FR = [int(lower_overlap[self.flag_355_total_FR][0]), self.polly_config_dict['maxDecomHeight355']] reference_search_range_532_total_FR = [int(lower_overlap[self.flag_532_total_FR][0]), self.polly_config_dict['maxDecomHeight532']] reference_search_range_1064_total_FR = [int(lower_overlap[self.flag_1064_total_FR][0]), self.polly_config_dict['maxDecomHeight1064']] self.polly_config_dict['reference_search_range_355_total_FR'] = reference_search_range_355_total_FR self.polly_config_dict['reference_search_range_532_total_FR'] = reference_search_range_532_total_FR self.polly_config_dict['reference_search_range_1064_total_FR'] = reference_search_range_1064_total_FR
# def __str__(self): # return f"{self.rawdata_dict}" def __del__(self): type(self).counter -= 1
[docs] def runProcessing(self): """Idea: Include a run function in the object as an alternative to the default run script... """ raise NotImplementedError()