BarrelAlignmentSpec.cxx

// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
// All rights not expressly granted are reserved.
//
// This software is distributed under the terms of the GNU General Public
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
//
// In applying this license CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.

/// @file   BarrelAlignmentSpec.cxx

#include <vector>
#include <string>
#include <filesystem>
#include <TMethodCall.h>
#include <TStopwatch.h>
#include <TROOT.h>
#include <TSystem.h>
#include "TMethodCall.h"
#include "AlignmentWorkflow/BarrelAlignmentSpec.h"
#include "Align/AlignableDetectorITS.h"
#include "Align/Controller.h"
#include "Align/AlignConfig.h"
#include "DetectorsBase/GRPGeomHelper.h"
#include "DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h"
#include "DataFormatsGlobalTracking/RecoContainer.h"
#include "DataFormatsParameters/GRPObject.h"
#include "CommonDataFormat/InteractionRecord.h"
#include "ReconstructionDataFormats/GlobalTrackID.h"
#include "CommonUtils/NameConf.h"
#include "DetectorsBase/Propagator.h"
#include "DetectorsBase/GeometryManager.h"
#include "DataFormatsITSMFT/TopologyDictionary.h"
#include "TRDBase/TrackletTransformer.h"
#include "CommonUtils/TreeStreamRedirector.h"
#include "TPCCalibration/VDriftHelper.h"
#include "TPCCalibration/CorrectionMapsLoader.h"
#include "GPUO2ExternalUser.h"
#include "GPUO2InterfaceUtils.h"
#include "GPUParam.h"
#include "Headers/DataHeader.h"
#include "Framework/ConfigParamRegistry.h"
#include "Framework/Task.h"
#include "Framework/CCDBParamSpec.h"
#include "Framework/ControlService.h"
#include "Framework/DeviceSpec.h"

#include <boost/interprocess/sync/named_semaphore.hpp>

/*
#include "DataFormatsITSMFT/TopologyDictionary.h"
#include "DataFormatsTPC/Constants.h"
#include "ReconstructionDataFormats/GlobalTrackAccessor.h"
#include "ReconstructionDataFormats/TrackTPCITS.h"
#include "ReconstructionDataFormats/MatchInfoTOF.h"
#include "ReconstructionDataFormats/TrackTPCTOF.h"
#include "DataFormatsITS/TrackITS.h"
#include "DataFormatsITSMFT/CompCluster.h"
#include "DataFormatsITSMFT/ROFRecord.h"
#include "DataFormatsITSMFT/TopologyDictionary.h"
#include "DataFormatsTPC/TrackTPC.h"
#include "DataFormatsTPC/ClusterNative.h"
#include "DataFormatsTPC/WorkflowHelper.h"
#include "ITSBase/GeometryTGeo.h"
#include "ITSMFTBase/DPLAlpideParam.h"
*/

using namespace o2::framework;
using namespace o2::globaltracking;
using namespace o2::align;

using GTrackID = o2::dataformats::GlobalTrackID;
using DetID = o2::detectors::DetID;

namespace o2
{
namespace align
{

class BarrelAlignmentSpec : public Task
{
 public:
  enum PostProc { WriteResults = 0x1 << 0,
                  CheckConstaints = 0x1 << 1,
                  GenPedeFiles = 0x1 << 2,
                  LabelPedeResults = 0x1 << 3 };
  BarrelAlignmentSpec(GTrackID::mask_t srcMP, std::shared_ptr<DataRequest> dr, std::shared_ptr<o2::base::GRPGeomRequest> ggrec, const o2::tpc::CorrectionMapsLoaderGloOpts& tpcOpt,
                      DetID::mask_t detmask, bool cosmic, int postprocess, bool useMC, bool loadTPCCalib)
    : mDataRequest(dr), mGRPGeomRequest(ggrec), mMPsrc{srcMP}, mDetMask{detmask}, mCosmic(cosmic), mPostProcessing(postprocess), mUseMC(useMC), mLoadTPCCalib(loadTPCCalib)
  {
    mTPCCorrMapsLoader.setLumiScaleType(tpcOpt.lumiType);
    mTPCCorrMapsLoader.setLumiScaleMode(tpcOpt.lumiMode);
    mTPCCorrMapsLoader.setCheckCTPIDCConsistency(tpcOpt.checkCTPIDCconsistency);
  }
  ~BarrelAlignmentSpec() override = default;
  void init(InitContext& ic) final;
  void run(ProcessingContext& pc) final;
  void finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj) final;
  void endOfStream(framework::EndOfStreamContext& ec) final;

 private:
  void updateTimeDependentParams(ProcessingContext& pc);
  std::string mIniParFile{};
  bool mUseIniParErrors = true;
  bool mUseMC = false;
  bool mIgnoreCCDBAlignment = false;
  bool mCosmic = false;
  bool mLoadTPCCalib = false;
  int mLane = 0;
  int mPostProcessing = 0; // special mode of extracting alignment or constraints check
  GTrackID::mask_t mMPsrc{};
  DetID::mask_t mDetMask{};
  std::unique_ptr<Controller> mController;
  std::shared_ptr<DataRequest> mDataRequest;
  std::shared_ptr<o2::base::GRPGeomRequest> mGRPGeomRequest;
  std::string mConfMacro{};
  std::unique_ptr<TMethodCall> mUsrConfMethod;
  std::unique_ptr<o2::trd::TrackletTransformer> mTRDTransformer;
  std::unique_ptr<o2::utils::TreeStreamRedirector> mDBGOut;
  std::unique_ptr<o2::gpu::GPUParam> mTPCParam;

  o2::tpc::VDriftHelper mTPCVDriftHelper{};
  o2::tpc::CorrectionMapsLoader mTPCCorrMapsLoader{};

  //
  TStopwatch mTimer;
};

void BarrelAlignmentSpec::init(InitContext& ic)
{
  mTimer.Stop();
  mTimer.Reset();
  o2::base::GRPGeomHelper::instance().setRequest(mGRPGeomRequest);

  int dbg = ic.options().get<int>("debug-output");
  mLane = ic.services().get<const o2::framework::DeviceSpec>().inputTimesliceId;
  mController = std::make_unique<Controller>(mDetMask, mMPsrc, mCosmic, mUseMC, mLane);
  if (dbg) {
    mController->setDebugOutputLevel(dbg);
  }

  mConfMacro = ic.options().get<std::string>("config-macro");
  if (!mConfMacro.empty()) {
    if (!std::filesystem::exists(mConfMacro)) {
      LOG(fatal) << "Requested user macro " << mConfMacro << " does not exist";
    }
    std::string tmpmacro = mConfMacro + "+";
    TString cmd = gSystem->GetMakeSharedLib();
    cmd += " -O0 -g -ggdb";
    // protect macro compilation by semaphore (to avoid problems in the pipelined code)
    {
      boost::interprocess::named_semaphore* sem = nullptr;
      std::string semhashedstring{};
      std::hash<std::string> hasher;
      semhashedstring = "align_macro_" + std::to_string(hasher(mConfMacro)).substr(0, 16);
      try {
        sem = new boost::interprocess::named_semaphore(boost::interprocess::open_or_create_t{}, semhashedstring.c_str(), 1);
      } catch (std::exception e) {
        LOGP(error, "Exception occurred during {} compilation semaphore setup", tmpmacro);
        sem = nullptr;
      }
      if (sem) {
        sem->wait(); // wait until we can enter (no one else there)
      }
      gSystem->SetMakeSharedLib(cmd.Data());
      auto res = gROOT->LoadMacro(tmpmacro.c_str());
      if (sem) {
        sem->post();
        if (sem->try_wait()) { // if nobody else is waiting remove the semaphore resource
          sem->post();
          boost::interprocess::named_semaphore::remove(semhashedstring.c_str());
        }
      }
      if (res) {
        LOG(fatal) << "Failed to load user macro " << tmpmacro;
      }
    }
    std::filesystem::path mpth(mConfMacro);
    mConfMacro = mpth.stem();
    mUsrConfMethod = std::make_unique<TMethodCall>();
    mUsrConfMethod->InitWithPrototype(mConfMacro.c_str(), "o2::align::Controller*, int");
  }
  mIgnoreCCDBAlignment = ic.options().get<bool>("ignore-ccdb-alignment");
  if (!mPostProcessing) {
    if (mLoadTPCCalib) {
      mTPCCorrMapsLoader.init(ic);
    }
    if (GTrackID::includesDet(DetID::TRD, mMPsrc)) {
      mTRDTransformer.reset(new o2::trd::TrackletTransformer);
      if (ic.options().get<bool>("apply-xor")) {
        mTRDTransformer->setApplyXOR();
      }
      auto prevShift = mTRDTransformer->isShiftApplied();
      if (getenv("ALIEN_JDL_LPMPRODUCTIONTYPE") && std::strcmp(getenv("ALIEN_JDL_LPMPRODUCTIONTYPE"), "MC") == 0) {
        // apply artificial pad shift in case non-ideal alignment is used to compensate for shift in current alignment from real data
        mTRDTransformer->setApplyShift(false);
      }
      LOGP(info, "Old TRD shift : {} new : {}", prevShift, mTRDTransformer->isShiftApplied());
      mController->setTRDTransformer(mTRDTransformer.get());
    }
    mController->setAllowAfterburnerTracks(ic.options().get<bool>("allow-afterburner-tracks"));
  }

  mIniParFile = ic.options().get<std::string>("initial-params-file");
  mUseIniParErrors = !ic.options().get<bool>("ignore-initial-params-errors");
  if (mPostProcessing && !(mPostProcessing != GenPedeFiles) && (mIniParFile.empty() || mIniParFile == "none")) {
    LOGP(warn, "Postprocessing {} is requested but the initial-params-file is not provided", mPostProcessing);
  }
}

void BarrelAlignmentSpec::updateTimeDependentParams(ProcessingContext& pc)
{
  o2::base::GRPGeomHelper::instance().checkUpdates(pc);
  auto tinfo = pc.services().get<o2::framework::TimingInfo>();
  if (pc.services().get<o2::framework::TimingInfo>().globalRunNumberChanged) {
    if (mController->getDebugOutputLevel()) {
      mDBGOut = std::make_unique<o2::utils::TreeStreamRedirector>(fmt::format("mpDebug_{}_{:08d}_{:010d}.root", mLane, tinfo.runNumber, tinfo.tfCounter).c_str(), "recreate");
      mController->setDebugStream(mDBGOut.get());
    }
    if (!mIgnoreCCDBAlignment) {
      for (auto id = DetID::First; id <= DetID::Last; id++) {
        const auto* alg = o2::base::GRPGeomHelper::instance().getAlignment(id);
        if (alg && !alg->empty()) {
          o2::base::GeometryManager::applyAlignment(*alg);
        }
        gGeoManager->RefreshPhysicalNodes(false);
      }
    } else {
      LOG(warn) << "CCDB alignment is NOT applied to ideal geometry";
    }
    if (!mController->getInitGeomDone()) {
      mController->initDetectors();
    }
    if (mTRDTransformer) { // need geometry loaded
      mTRDTransformer->init();
    }

    if (!(mIniParFile.empty() || mIniParFile == "none")) {
      mController->readParameters(mIniParFile, mUseIniParErrors);
      mController->applyAlignmentFromMPSol();
    }

    // call this in the very end
    if (mUsrConfMethod) {
      int dummyPar = 0, ret = -1;
      Controller* tmpPtr = mController.get();
      const void* args[2] = {&tmpPtr, &dummyPar};
      mUsrConfMethod->Execute(nullptr, args, 2, &ret);
      if (ret != 0) {
        LOG(fatal) << "Execution of user method config method " << mConfMacro << " failed with " << ret;
      }
    }
    AlignConfig::Instance().printKeyValues(true);
    o2::base::PropagatorD::Instance()->setTGeoFallBackAllowed(false);
  }
  if (GTrackID::includesDet(DetID::TRD, mMPsrc) && mTRDTransformer) {
    pc.inputs().get<o2::trd::CalVdriftExB*>("calvdexb"); // just to trigger the finaliseCCDB
  }
  if (mLoadTPCCalib) {

    static float prevField = 1e-6;
    float newField = o2::base::Propagator::Instance()->getNominalBz();
    if (prevField != newField) {
      prevField = newField;
      if (mDetMask[DetID::TPC]) {
        mTPCParam.reset(new o2::gpu::GPUParam);
        mTPCParam->SetDefaults(o2::base::Propagator::Instance()->getNominalBz(), false);
        mController->setTPCParam(mTPCParam.get());
      }
    }

    mTPCVDriftHelper.extractCCDBInputs(pc);
    mTPCCorrMapsLoader.extractCCDBInputs(pc);
    bool updateMaps = false;
    if (mTPCCorrMapsLoader.isUpdated()) {
      mTPCCorrMapsLoader.acknowledgeUpdate();
      updateMaps = true;
    }
    mController->setTPCCorrMaps(&mTPCCorrMapsLoader);
    if (mTPCVDriftHelper.isUpdated()) {
      LOGP(info, "Updating TPC fast transform map with new VDrift factor of {} wrt reference {} and DriftTimeOffset correction {} wrt {} from source {}",
           mTPCVDriftHelper.getVDriftObject().corrFact, mTPCVDriftHelper.getVDriftObject().refVDrift,
           mTPCVDriftHelper.getVDriftObject().timeOffsetCorr, mTPCVDriftHelper.getVDriftObject().refTimeOffset,
           mTPCVDriftHelper.getSourceName());
      mController->setTPCVDrift(mTPCVDriftHelper.getVDriftObject());
      mTPCVDriftHelper.acknowledgeUpdate();
      updateMaps = true;
    }
    if (updateMaps) {
      mTPCCorrMapsLoader.updateVDrift(mTPCVDriftHelper.getVDriftObject().corrFact, mTPCVDriftHelper.getVDriftObject().refVDrift, mTPCVDriftHelper.getVDriftObject().getTimeOffset());
    }
  }
}

void BarrelAlignmentSpec::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj)
{
  o2::base::GRPGeomHelper::instance().finaliseCCDB(matcher, obj);
  if (matcher == ConcreteDataMatcher("ITS", "CLUSDICT", 0)) {
    auto* its = mController->getDetector(o2::detectors::DetID::ITS);
    if (its) {
      LOG(info) << "cluster dictionary updated";
      ((AlignableDetectorITS*)its)->setITSDictionary((const o2::itsmft::TopologyDictionary*)obj);
      return;
    }
  }
  if (matcher == ConcreteDataMatcher("TRD", "CALVDRIFTEXB", 0)) {
    LOG(info) << "CalVdriftExB object has been updated";
    mTRDTransformer->setCalVdriftExB((const o2::trd::CalVdriftExB*)obj);
    return;
  }
  if (mTPCVDriftHelper.accountCCDBInputs(matcher, obj)) {

    return;
  }
  if (mTPCCorrMapsLoader.accountCCDBInputs(matcher, obj)) {
    return;
  }
}

void BarrelAlignmentSpec::run(ProcessingContext& pc)
{
  mTimer.Start(false);
  if (mPostProcessing) { // special mode, no data processing
    updateTimeDependentParams(pc);
    if (mController->getInstanceID() == 0) {
      if (mPostProcessing & PostProc::CheckConstaints) {
        mController->addAutoConstraints();
        mController->checkConstraints();
      }
      if (mPostProcessing & PostProc::WriteResults) {
        mController->writeCalibrationResults();
      }
    }
    pc.services().get<o2::framework::ControlService>().readyToQuit(framework::QuitRequest::Me);
  } else {
    RecoContainer recoData;
    recoData.collectData(pc, *mDataRequest.get());
    updateTimeDependentParams(pc); // call after collectData !!!
    mController->setRecoContainer(&recoData);
    mController->setTimingInfo(pc.services().get<o2::framework::TimingInfo>());
    if (mCosmic) {
      mController->processCosmic();
    } else {
      mController->process();
    }
  }
  mTimer.Stop();
}

void BarrelAlignmentSpec::endOfStream(EndOfStreamContext& ec)
{
  auto inst = ec.services().get<const o2::framework::DeviceSpec>().inputTimesliceId;
  if (!mPostProcessing) {
    LOGP(info, "Barrel alignment data pereparation total timing: Cpu: {:.3e} Real: {:.3e}  s in {} slots, instance {}", mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1, inst);
    mController->closeMPRecOutput();
    mController->closeMilleOutput();
    mController->closeResidOutput();
  }
  if (inst == 0) {
    if (!mPostProcessing || (mPostProcessing & PostProc::GenPedeFiles)) {
      LOG(info) << "Writing millepede control files";
      if (!mPostProcessing) {
        mController->terminate(); // finalize data stat
      }
      mController->addAutoConstraints();
      mController->genPedeSteerFile();
      mController->getStat().print();
    } else if (mPostProcessing & PostProc::LabelPedeResults) {
      mController->writeLabeledPedeResults();
    }
  }
  mDBGOut.reset();
}

DataProcessorSpec getBarrelAlignmentSpec(GTrackID::mask_t srcMP, GTrackID::mask_t src, DetID::mask_t dets, DetID::mask_t skipDetClusters, bool enableCosmic, int postprocess, bool useMC, const o2::tpc::CorrectionMapsLoaderGloOpts& sclOpts)
{
  std::vector<OutputSpec> outputs;
  auto dataRequest = std::make_shared<DataRequest>();
  bool loadTPCCalib = false;
  Options opts{
    ConfigParamSpec{"apply-xor", o2::framework::VariantType::Bool, false, {"flip the 8-th bit of slope and position (for processing TRD CTFs from 2021 pilot beam)"}},
    ConfigParamSpec{"allow-afterburner-tracks", VariantType::Bool, false, {"allow using ITS-TPC afterburner tracks"}},
    ConfigParamSpec{"ignore-ccdb-alignment", VariantType::Bool, false, {"do not aplly CCDB alignment to ideal geometry"}},
    ConfigParamSpec{"initial-params-file", VariantType::String, "", {"initial parameters file"}},
    ConfigParamSpec{"config-macro", VariantType::String, "", {"configuration macro with signature (o2::align::Controller*, int) to execute from init"}},
    ConfigParamSpec{"ignore-initial-params-errors", VariantType::Bool, false, {"ignore initial params (if any) errors for precondition"}},
    ConfigParamSpec{"debug-output", VariantType::Int, 0, {"produce debugging output root files"}}};
  if (!postprocess) {
    dataRequest->requestTracks(src, useMC);
    dataRequest->requestClusters(src, false, skipDetClusters);
    dataRequest->requestPrimaryVertices(useMC);
    if (GTrackID::includesDet(DetID::TRD, srcMP)) {
      dataRequest->inputs.emplace_back("calvdexb", "TRD", "CALVDRIFTEXB", 0, Lifetime::Condition, ccdbParamSpec("TRD/Calib/CalVdriftExB"));
    }
    if (enableCosmic) {
      dataRequest->requestCoscmicTracks(useMC);
    }
    if (src[DetID::TPC] && !skipDetClusters[DetID::TPC]) {
      o2::tpc::VDriftHelper::requestCCDBInputs(dataRequest->inputs);
      o2::tpc::CorrectionMapsLoader::requestCCDBInputs(dataRequest->inputs, opts, sclOpts);
      loadTPCCalib = true;
    }
  }
  auto ccdbRequest = std::make_shared<o2::base::GRPGeomRequest>(true,                                 // orbitResetTime
                                                                true,                                 // GRPECS=true
                                                                false,                                // GRPLHCIF
                                                                true,                                 // GRPMagField
                                                                true,                                 // askMatLUT
                                                                o2::base::GRPGeomRequest::Alignments, // geometry
                                                                dataRequest->inputs,
                                                                false,              // ask update once (except field)
                                                                true,               // init PropagatorD
                                                                "ITS,TPC,TRD,TOF"); // alignment objects to apply
  return DataProcessorSpec{
    "barrel-alignment",
    dataRequest->inputs,
    outputs,
    AlgorithmSpec{adaptFromTask<BarrelAlignmentSpec>(srcMP, dataRequest, ccdbRequest, sclOpts, dets, enableCosmic, postprocess, useMC, loadTPCCalib)},
    opts};
}

} // namespace align
} // namespace o2