main.cpp

Go to the documentation of this file.

// -*-Mode: C++;-*-

// * BeginRiceCopyright *****************************************************
//
// $HeadURL$
// $Id$
//
// --------------------------------------------------------------------------
// Part of HPCToolkit (hpctoolkit.org)
//
// Information about sources of support for research and development of
// HPCToolkit is at 'hpctoolkit.org' and in 'README.Acknowledgments'.
// --------------------------------------------------------------------------
//
// Copyright ((c)) 2002-2019, Rice University
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
//   notice, this list of conditions and the following disclaimer.
//
// * Redistributions in binary form must reproduce the above copyright
//   notice, this list of conditions and the following disclaimer in the
//   documentation and/or other materials provided with the distribution.
//
// * Neither the name of Rice University (RICE) nor the names of its
//   contributors may be used to endorse or promote products derived from
//   this software without specific prior written permission.
//
// This software is provided by RICE and contributors "as is" and any
// express or implied warranties, including, but not limited to, the
// implied warranties of merchantability and fitness for a particular
// purpose are disclaimed. In no event shall RICE or contributors be
// liable for any direct, indirect, incidental, special, exemplary, or
// consequential damages (including, but not limited to, procurement of
// substitute goods or services; loss of use, data, or profits; or
// business interruption) however caused and on any theory of liability,
// whether in contract, strict liability, or tort (including negligence
// or otherwise) arising in any way out of the use of this software, even
// if advised of the possibility of such damage.
//
// ******************************************************* EndRiceCopyright *

//***************************************************************************
//
// File:
//   $HeadURL$
//
// Purpose:
//   [The purpose of this file]
//
// Description:
//   [The set of functions, macros, etc. defined in the file]
//
//***************************************************************************

//**************************** MPI Include Files ****************************

#include <mpi.h>

//************************* System Include Files ****************************

#include <iostream>
#include <fstream>
#include <typeinfo>
#include <signal.h>
#include <unistd.h>

#include <string>
using std::string;

#include <vector>
using std::vector;

#include <cstdlib> // getenv()
#include <cmath>   // ceil()
#include <climits> // UCHAR_MAX, PATH_MAX
#include <cctype>  // isdigit()
#include <cstring> // strcpy()

//*************************** User Include Files ****************************

#include <include/uint.h>

#include "Args.hpp"
#include "ParallelAnalysis.hpp"

#include <lib/analysis/CallPath.hpp>
#include <lib/analysis/Util.hpp>

#include <lib/binutils/VMAInterval.hpp>
#include <lib/prof/FileError.hpp>

#include <lib/prof-lean/hpcrun-fmt.h>

#include <lib/support/diagnostics.h>
#include <lib/support/FileUtil.hpp>
#include <lib/support/Logic.hpp>
#include <lib/support/RealPathMgr.hpp>
#include <lib/support/StrUtil.hpp>

#define DBG_PROF_MPI 0

//*************************** Forward Declarations ***************************

static int
realmain(int argc, char* const* argv);

static Analysis::Util::NormalizeProfileArgs_t
myNormalizeProfileArgs(const Analysis::Util::StringVec& profileFiles,
                       vector<uint>& groupIdToGroupSizeMap,
                       int myRank, int numRanks);


static void
makeSummaryMetrics(Prof::CallPath::Profile& profGbl,
                   const Analysis::Args& args,
                   const Analysis::Util::NormalizeProfileArgs_t& nArgs,
                   const vector<uint>& groupIdToGroupSizeMap,
                   int myRank, int numRanks);

static void
makeThreadMetrics(Prof::CallPath::Profile& profGbl,
                  const Analysis::Args& args,
                  const Analysis::Util::NormalizeProfileArgs_t& nArgs,
                  const vector<uint>& groupIdToGroupSizeMap,
                  int myRank, int numRanks);

static uint
makeDerivedMetricDescs(Prof::CallPath::Profile& profGbl,
                       const Analysis::Args& args,
                       uint& mDrvdBeg, uint& mDrvdEnd,
                       uint& mXDrvdBeg, uint& mXDrvdEnd,
                       vector<VMAIntervalSet*>& groupIdToGroupMetricsMap,
                       const vector<uint>& groupIdToGroupSizeMap,
                       int myRank);

static void
makeSummaryMetrics_Lcl(Prof::CallPath::Profile& profGbl,
                       const string& profileFile,
                       const Analysis::Args& args, uint groupId, uint groupMax,
                       vector<VMAIntervalSet*>& groupIdToGroupMetricsMap,
                       int myRank);

static void
makeThreadMetrics_Lcl(Prof::CallPath::Profile& profGbl,
                      const string& profileFile,
                      const Analysis::Args& args, uint groupId, uint groupMax,
                      int myRank);

static string
makeDBFileName(const string& dbDir, uint groupId, const string& profileFile);

static void
writeMetricsDB(Prof::CallPath::Profile& profGbl, uint mBegId, uint mEndId,
               const string& metricDBFnm);


static void
writeStructure(const Prof::Struct::Tree& structure, const char* baseNm,
               int myRank) __attribute__((unused));

static void
writeProfile(const Prof::CallPath::Profile& prof, const char* baseNm,
             int myRank) __attribute__((unused));

static std::string
makeFileName(const char* baseNm, const char* ext, int myRank);


//****************************************************************************

void 
prof_abort
(
  int error_code
)
{
  MPI_Abort(MPI_COMM_WORLD, error_code);
}


int 
main(int argc, char* const* argv) 
{
  int ret;

  try {
    ret = realmain(argc, argv);
  }
  catch (const Diagnostics::Exception& x) {
    DIAG_EMsg(x.message());
    exit(1);
  }
  catch (const std::bad_alloc& x) {
    DIAG_EMsg("[std::bad_alloc] " << x.what());
    exit(1);
  }
  catch (const std::exception& x) {
    DIAG_EMsg("[std::exception] " << x.what());
    exit(1);
  }
  catch (...) {
    DIAG_EMsg("Unknown exception encountered!");
    exit(2);
  }

  return ret;
}


static void
abort_timeout_handler(int sig, siginfo_t* siginfo, void* context)
{
  abort();
}

static void
hpcprof_set_abort_timeout()
{
  char *error_timeout = getenv("HPCPROF_ABORT_TIMEOUT");
  if (error_timeout) {
     int seconds = atoi(error_timeout);
     if (seconds != 0) {
       struct sigaction act;
       act.sa_sigaction = abort_timeout_handler;
       act.sa_flags = SA_SIGINFO;
       sigaction(SIGALRM, &act, NULL);
       alarm(seconds);
     }
  }
}


static int
realmain(int argc, char* const* argv) 
{
  Args args;
  args.parse(argc, argv); // may call exit()

  RealPathMgr::singleton().searchPaths(args.searchPathStr());
  hpcprof_set_abort_timeout();

  // -------------------------------------------------------
  // 0. MPI initialize
  // -------------------------------------------------------
  MPI_Init(&argc, (char***)&argv);

  int myRank, numRanks;
  MPI_Comm_rank(MPI_COMM_WORLD, &myRank); 
  MPI_Comm_size(MPI_COMM_WORLD, &numRanks);

  // -------------------------------------------------------
  // 0. Debugging hook
  // -------------------------------------------------------
  const char* HPCPROF_WAIT = getenv("HPCPROF_WAIT");
  if (HPCPROF_WAIT) {
    int waitRank = 0;
    if (strlen(HPCPROF_WAIT) > 0) {
      waitRank = atoi(HPCPROF_WAIT);
    }

    volatile int DEBUGGER_WAIT = 1;
    if (myRank == waitRank) {
      while (DEBUGGER_WAIT);
    }
  }

  // -------------------------------------------------------
  // 0. Make empty Experiment database (ensure file system works)
  // -------------------------------------------------------
  if (myRank == 0)
    args.makeDatabaseDir();

  char dbDirBuf[PATH_MAX];
  if (myRank == 0) {
    memset(dbDirBuf, '\0', PATH_MAX); // avoid artificial valgrind warnings
    strncpy(dbDirBuf, args.db_dir.c_str(), PATH_MAX);
    dbDirBuf[PATH_MAX - 1] = '\0';
  }

  MPI_Bcast((void*)dbDirBuf, PATH_MAX, MPI_CHAR, 0, MPI_COMM_WORLD);
  args.db_dir = dbDirBuf;

  // -------------------------------------------------------
  // 1a. Create local CCT (from local set of profile files)
  // -------------------------------------------------------
  Prof::CallPath::Profile* profLcl = NULL;

  vector<uint> groupIdToGroupSizeMap; // only initialized for rank 0

  Analysis::Util::NormalizeProfileArgs_t nArgs =
    myNormalizeProfileArgs(args.profileFiles, groupIdToGroupSizeMap,
                           myRank, numRanks);

  if (nArgs.paths->size() == 0 && myRank == 0) {
    std::cerr << "ERROR: command line directories"
      " contain no .hpcrun files; no database generated\n";
    prof_abort(-1);
  }

  int mergeTy = Prof::CallPath::Profile::Merge_MergeMetricByName;
  uint rFlags = (Prof::CallPath::Profile::RFlg_VirtualMetrics
                 | Prof::CallPath::Profile::RFlg_NoMetricSfx
                 | Prof::CallPath::Profile::RFlg_MakeInclExcl);
  Analysis::Util::UIntVec* groupMap =
    (nArgs.groupMax > 1) ? nArgs.groupMap : NULL;

  profLcl = Analysis::CallPath::read(*nArgs.paths, groupMap, mergeTy, rFlags);

  // -------------------------------------------------------
  // 1b. Create canonical CCT (metrics merged by <group>.<name>.*)
  // -------------------------------------------------------
  Prof::CallPath::Profile* profGbl = NULL;

  // Post-INVARIANT: rank 0's 'profLcl' is the canonical CCT.  Metrics
  // are merged (and sorted by always merging left-child before right)
  ParallelAnalysis::reduce(profLcl, myRank, numRanks);

  ParallelAnalysis::reduce(&profLcl->directorySet(), myRank, numRanks);

  if (myRank == 0) {
    profGbl = profLcl;
    profLcl = NULL;
  }

  // Post-INVARIANT: 'profGbl' is the canonical CCT
  ParallelAnalysis::broadcast(profGbl, myRank);

  if (myRank == 0) {
    profGbl->metricMgr()->mergePerfEventStatistics_finalize(numRanks - 1);
  }

  ParallelAnalysis::broadcast(profGbl->directorySet(), myRank);

  delete profLcl;

  // -------------------------------------------------------
  // 1c. Add static structure to canonical CCT; form dense node ids
  //
  // Post-INVARIANT: each process has canonical CCT with dense node
  // ids; corresponding nodes have idential ids.
  // -------------------------------------------------------

  Prof::Struct::Tree* structure = new Prof::Struct::Tree("");
  if (!args.structureFiles.empty()) {
    Analysis::CallPath::readStructure(structure, args);
  }
  profGbl->structure(structure);


  // N.B.: Ensures that each rank adds static structure in the same
  // order so that new corresponding nodes have identical node ids.
  bool printProgress =  (myRank == 0);
  Analysis::CallPath::overlayStaticStructureMain(*profGbl, args.agent,
                                                 args.doNormalizeTy,
                                                 printProgress);

  // N.B.: Dense ids are assigned w.r.t. Prof::CCT::...::cmpByStructureInfo()
  DIAG_MsgIf(DBG_PROF_MPI, myRank << ". numcct before: " << profGbl->cct()->maxDenseId() << std::endl);

  profGbl->cct()->makeDensePreorderIds();

  DIAG_MsgIf(DBG_PROF_MPI, myRank << ". numcct after: " << profGbl->cct()->maxDenseId() << std::endl);

  // -------------------------------------------------------
  // 2a. Create summary metrics for canonical CCT
  //
  // Post-INVARIANT: rank 0's 'profGbl' contains summary metrics
  // -------------------------------------------------------
  DIAG_MsgIf(DBG_PROF_MPI, myRank << ". stage 2a: makeSummaryMetrics" << std::endl);

  makeSummaryMetrics(*profGbl, args, nArgs, groupIdToGroupSizeMap,
                     myRank, numRanks);

  // -------------------------------------------------------
  // 2b. Prune and normalize canonical CCT
  // -------------------------------------------------------
  DIAG_MsgIf(DBG_PROF_MPI, myRank << ". stage 2b: pruneBySummaryMetrics" << std::endl);

  uint prunedNodesSz = profGbl->cct()->maxDenseId() + 1;
  uint8_t* prunedNodes = new uint8_t[prunedNodesSz];
  memset(prunedNodes, 0, prunedNodesSz * sizeof(uint8_t));

  if (myRank == 0) {
    // Disable pruning when making a metric database because it causes
    // makeThreadMetrics_Lcl(), which uses CCT::MrgFlg_CCTMergeOnly,
    // to under-compute values for thread-level metrics.
    if (!args.db_makeMetricDB) {
      Analysis::CallPath::pruneBySummaryMetrics(*profGbl, prunedNodes);
    }
  }
  
  DIAG_MsgIf(DBG_PROF_MPI, myRank << ". stage 2b1: broadcast prunedNodes" << std::endl);

  MPI_Bcast(prunedNodes, prunedNodesSz, MPI_BYTE, 0, MPI_COMM_WORLD);

  if (myRank != 0) {
    profGbl->cct()->pruneCCTByNodeId(prunedNodes);
  }
  delete[] prunedNodes;

  DIAG_MsgIf(DBG_PROF_MPI, myRank << ". stage 2b2: normalize" << std::endl);

  Analysis::CallPath::normalize(*profGbl, args.agent, args.doNormalizeTy);

  if (myRank == 0) {
    // Apply after all CCT pruning/normalization is completed.
    Analysis::CallPath::applySummaryMetricAgents(*profGbl, args.agent);
  }

  // N.B.: Dense ids are assigned w.r.t. Prof::CCT::...::cmpByStructureInfo()
  profGbl->cct()->makeDensePreorderIds();

  // -------------------------------------------------------
  // 2c. Create thread-level metric DB // Normalize trace files
  // -------------------------------------------------------
  DIAG_MsgIf(DBG_PROF_MPI, myRank << ". stage 2c: makeThreadMetrics" << std::endl);

  makeThreadMetrics(*profGbl, args, nArgs, groupIdToGroupSizeMap,
                    myRank, numRanks);
  
  // ------------------------------------------------------------
  // 3. Generate Experiment database
  //    INVARIANT: database dir already exists
  // ------------------------------------------------------------
  DIAG_MsgIf(DBG_PROF_MPI, myRank << ". stage 3: pruneStructTree" << std::endl);

  Analysis::CallPath::pruneStructTree(*profGbl);

  if (myRank == 0) {
    if (args.title.empty()) {
      args.title = profGbl->name();
    }

    if (!args.db_makeMetricDB) {
      profGbl->metricMgr()->zeroDBInfo();
    }

    Analysis::CallPath::makeDatabase(*profGbl, args);
  }
  else {
    Analysis::Util::copyTraceFiles(args.db_dir, profGbl->traceFileNameSet());
  }

  // -------------------------------------------------------
  // Cleanup/MPI finalize
  // -------------------------------------------------------
  nArgs.destroy();

  delete profGbl;

  MPI_Finalize();

  return 0;
}


//****************************************************************************

// myNormalizeProfileArgs: creates canonical list of profiles files and
//   distributes chunks of size ceil(numFiles / numRanks) to each process.
//   The last process may have a smaller chunk than the others.
static Analysis::Util::NormalizeProfileArgs_t
myNormalizeProfileArgs(const Analysis::Util::StringVec& profileFiles,
                       vector<uint>& groupIdToGroupSizeMap,
                       int myRank, int numRanks)
{
  Analysis::Util::NormalizeProfileArgs_t out;

  char* sendFilesBuf = NULL;
  uint sendFilesBufSz = 0;
  uint sendFilesChunkSz = 0;
  const uint groupIdLen = 1; // see asssertion below
  uint pathLenMax = 0;
  uint groupIdMax = 0;

  // -------------------------------------------------------
  // root creates canonical and grouped list of files
  // -------------------------------------------------------

  if (myRank == 0) {
    Analysis::Util::NormalizeProfileArgs_t nArgs =
      Analysis::Util::normalizeProfileArgs(profileFiles);
    
    Analysis::Util::StringVec* canonicalFiles = nArgs.paths;
    pathLenMax = nArgs.pathLenMax;
    groupIdMax = nArgs.groupMax;

    DIAG_Assert(nArgs.groupMax <= UCHAR_MAX, "myNormalizeProfileArgs: 'groupMax' cannot be packed into a uchar!");

    uint chunkSz = (uint)
      ceil( (double)canonicalFiles->size() / (double)numRanks);
    
    sendFilesChunkSz = chunkSz * (groupIdLen + pathLenMax + 1);
    sendFilesBufSz = sendFilesChunkSz * numRanks;
    sendFilesBuf = new char[sendFilesBufSz];
    memset(sendFilesBuf, '\0', sendFilesBufSz);

    groupIdToGroupSizeMap.resize(groupIdMax + 1);
    
    for (uint i = 0, j = 0; i < canonicalFiles->size(); 
         i++, j += (groupIdLen + pathLenMax + 1)) {
      const std::string& nm = (*canonicalFiles)[i];
      uint groupId = (*nArgs.groupMap)[i];

      groupIdToGroupSizeMap[groupId]++;

      // pack into sendFilesBuf
      sendFilesBuf[j] = (char)groupId;
      strncpy(&(sendFilesBuf[j + groupIdLen]), nm.c_str(), pathLenMax);
      sendFilesBuf[j + groupIdLen + pathLenMax] = '\0';
    }

    nArgs.destroy();
  }

  // -------------------------------------------------------
  // prepare parameters for scatter
  // -------------------------------------------------------
  
  const uint metadataBufSz = 3;
  uint metadataBuf[metadataBufSz];
  metadataBuf[0] = sendFilesChunkSz;
  metadataBuf[1] = pathLenMax;
  metadataBuf[2] = groupIdMax;

  MPI_Bcast((void*)metadataBuf, metadataBufSz, MPI_UNSIGNED,
            0, MPI_COMM_WORLD);

  if (myRank != 0) {
    sendFilesChunkSz = metadataBuf[0];
    pathLenMax       = metadataBuf[1];
    groupIdMax       = metadataBuf[2];
  }

  // -------------------------------------------------------
  // evenly distribute profile files across all processes
  // -------------------------------------------------------

  uint recvFilesChunkSz = sendFilesChunkSz;
  const char* recvFilesBuf = new char[recvFilesChunkSz];
  
  MPI_Scatter((void*)sendFilesBuf, sendFilesChunkSz, MPI_CHAR,
              (void*)recvFilesBuf, recvFilesChunkSz, MPI_CHAR,
              0, MPI_COMM_WORLD);
  
  delete[] sendFilesBuf;


  for (uint i = 0; i < recvFilesChunkSz; i += (groupIdLen + pathLenMax + 1)) {
    uint groupId = recvFilesBuf[i];
    const char* nm_cstr = &recvFilesBuf[i + groupIdLen];
    string nm = nm_cstr;
    if (!nm.empty()) {
      out.paths->push_back(nm);
      out.groupMap->push_back(groupId);
    }
  }

  out.pathLenMax = pathLenMax;
  out.groupMax = groupIdMax;

  delete[] recvFilesBuf;

  if (0) {
    for (uint i = 0; i < out.paths->size(); ++i) {
      const std::string& nm = (*out.paths)[i];
      std::cout << "[" << myRank << "]: " << nm << std::endl;
    }
  }

  return out;
}


//***************************************************************************

// makeSummaryMetrics: Assumes 'profGbl' is the canonical CCT (with
// structure and with canonical ids).
static void
makeSummaryMetrics(Prof::CallPath::Profile& profGbl,
                   const Analysis::Args& args,
                   const Analysis::Util::NormalizeProfileArgs_t& nArgs,
                   const vector<uint>& groupIdToGroupSizeMap,
                   int myRank, int numRanks)
{
  uint mDrvdBeg = 0, mDrvdEnd = 0;   // [ )
  uint mXDrvdBeg = 0, mXDrvdEnd = 0; // [ )

  vector<VMAIntervalSet*> groupIdToGroupMetricsMap(nArgs.groupMax + 1, NULL);

  makeDerivedMetricDescs(profGbl, args,
                         mDrvdBeg, mDrvdEnd, mXDrvdBeg, mXDrvdEnd,
                         groupIdToGroupMetricsMap, groupIdToGroupSizeMap,
                         myRank);

  Prof::Metric::Mgr& mMgrGbl = *profGbl.metricMgr();
  Prof::CCT::ANode* cctRoot = profGbl.cct()->root();

  // -------------------------------------------------------
  // compute local contribution summary metrics (accumulate function)
  // -------------------------------------------------------
  cctRoot->computeMetricsIncr(mMgrGbl, mDrvdBeg, mDrvdEnd,
                              Prof::Metric::AExprIncr::FnInit);

  for (uint i = 0; i < nArgs.paths->size(); ++i) {
    const string& fnm = (*nArgs.paths)[i];
    uint groupId = (*nArgs.groupMap)[i];
    makeSummaryMetrics_Lcl(profGbl, fnm, args, groupId, nArgs.groupMax,
                           groupIdToGroupMetricsMap, myRank);
  }

  // -------------------------------------------------------
  // create summary metrics via reduction (combine function)
  // -------------------------------------------------------

  // 1. Change definitions of derived metrics [mDrvdBeg, mDrvdEnd)
  //    since the 'combine' function will be used during the metric
  //    reduction.  Metric inputs point to accumulators [mXDrvdBeg,
  //    mXDrvdEnd) rather input values.
  for (uint i = mDrvdBeg, j = mXDrvdBeg; i < mDrvdEnd; ++i) {
    Prof::Metric::ADesc* m = mMgrGbl.metric(i);
    Prof::Metric::DerivedIncrDesc* mm =
      dynamic_cast<Prof::Metric::DerivedIncrDesc*>(m);
    DIAG_Assert(mm, DIAG_UnexpectedInput);

    Prof::Metric::AExprIncr* expr = mm->expr();
    if (expr) {
      for (uint k = 0; k < expr->numAccum(); ++k)
        expr->srcId(k, j++);
    }
  }

  // 2. Initialize temporary accumulators [mXDrvdBeg, mXDrvdEnd) used
  //    during the summary metrics reduction
  cctRoot->computeMetricsIncr(mMgrGbl, mXDrvdBeg, mXDrvdEnd,
                              Prof::Metric::AExprIncr::FnInitSrc);

  // 3. Reduction
  uint maxCCTId = profGbl.cct()->maxDenseId();

  ParallelAnalysis::PackedMetrics* packedMetrics =
    new ParallelAnalysis::PackedMetrics(maxCCTId + 1, mXDrvdBeg, mXDrvdEnd,
                                        mDrvdBeg, mDrvdEnd);

  // Post-INVARIANT: rank 0's 'profGbl' contains summary metrics
  ParallelAnalysis::reduce(std::make_pair(&profGbl, packedMetrics),
                           myRank, numRanks);

  // -------------------------------------------------------
  // finalize metrics
  // -------------------------------------------------------

  if (myRank == 0) {
    
    for (uint i = 0; i < mMgrGbl.size(); ++i) {
      Prof::Metric::ADesc* m = mMgrGbl.metric(i);
      m->computedType(Prof::Metric::ADesc::ComputedTy_NonFinal);
    }
  }

  for (uint grpId = 1; grpId < groupIdToGroupMetricsMap.size(); ++grpId) {
    delete groupIdToGroupMetricsMap[grpId];
  }

  delete packedMetrics;
}


static void
makeThreadMetrics(Prof::CallPath::Profile& profGbl,
                  const Analysis::Args& args,
                  const Analysis::Util::NormalizeProfileArgs_t& nArgs,
                  const vector<uint>& groupIdToGroupSizeMap,
                  int myRank, int numRanks)
{
  for (uint i = 0; i < nArgs.paths->size(); ++i) {
    string& fnm = (*nArgs.paths)[i];
    uint groupId = (*nArgs.groupMap)[i];
    makeThreadMetrics_Lcl(profGbl, fnm, args, groupId, nArgs.groupMax, myRank);
  }
}


static uint
makeDerivedMetricDescs(Prof::CallPath::Profile& profGbl,
                       const Analysis::Args& args,
                       uint& mDrvdBeg, uint& mDrvdEnd,
                       uint& mXDrvdBeg, uint& mXDrvdEnd,
                       vector<VMAIntervalSet*>& groupIdToGroupMetricsMap,
                       const vector<uint>& groupIdToGroupSizeMap,
                       int myRank)
{
  Prof::Metric::Mgr& mMgrGbl = *(profGbl.metricMgr());

  uint numSrc = mMgrGbl.size();
  uint mSrcBeg = 0, mSrcEnd = numSrc; // [ )

  uint numDrvd = 0;
  mDrvdBeg = mDrvdEnd = 0;   // [ )
  mXDrvdBeg = mXDrvdEnd = 0; // [ )

  // -------------------------------------------------------
  // official set of derived metrics
  // -------------------------------------------------------

  bool needAllStats =
    Analysis::Args::MetricFlg_isSet(args.prof_metrics,
                                    Analysis::Args::MetricFlg_StatsAll);

  mDrvdBeg = mMgrGbl.makeSummaryMetricsIncr(needAllStats, mSrcBeg, mSrcEnd);
  if (mDrvdBeg != Prof::Metric::Mgr::npos) {
    mDrvdEnd = mMgrGbl.size();
    numDrvd = (mDrvdEnd - mDrvdBeg);
  }

  for (uint i = mSrcBeg; i < mSrcEnd; ++i) {
    Prof::Metric::ADesc* m = mMgrGbl.metric(i);
    m->isVisible(false);
  }

  for (uint i = mDrvdBeg; i < mDrvdEnd; ++i) {
    Prof::Metric::ADesc* m = mMgrGbl.metric(i);

    uint groupId = 1; // default group-id

    // find groupId embedded in metric descriptor name
    const string& nmPfx = m->namePfx();
    if (!nmPfx.empty()) {
      groupId = (uint)StrUtil::toUInt64(nmPfx);
    }
    DIAG_Assert(groupId > 0, DIAG_UnexpectedInput);
    DIAG_Assert(groupId < groupIdToGroupMetricsMap.size(), DIAG_UnexpectedInput);

    // rank 0: set the number of inputs
    if (myRank == 0) {
      Prof::Metric::DerivedIncrDesc* mm = 
        dynamic_cast<Prof::Metric::DerivedIncrDesc*>(m);
      DIAG_Assert(mm, DIAG_UnexpectedInput);
    
      // N.B.: groupIdToGroupSizeMap is only initialized for rank 0
      uint numInputs = groupIdToGroupSizeMap[groupId]; // / <n> TODO:threads
      if (mm->expr()) {
        mm->expr()->numSrcFxd(numInputs);
      }
    }
   
    // populate groupIdToGroupMetricsMap map
    VMAIntervalSet*& ivalset = groupIdToGroupMetricsMap[groupId];
    if (!ivalset) {
      ivalset = new VMAIntervalSet;
    }
    ivalset->insert(i, i + 1); // [ )
  }
  
  // -------------------------------------------------------
  // make temporary set of extra derived metrics (for reduction)
  // -------------------------------------------------------
  mXDrvdBeg = mMgrGbl.makeSummaryMetricsIncr(needAllStats, mSrcBeg, mSrcEnd);
  if (mXDrvdBeg != Prof::Metric::Mgr::npos) {
    mXDrvdEnd = mMgrGbl.size();
  }
  
  for (uint i = mXDrvdBeg; i < mXDrvdEnd; ++i) {
    Prof::Metric::ADesc* m = mMgrGbl.metric(i);
    m->isVisible(false);
  }

  profGbl.isMetricMgrVirtual(false);

  return numDrvd;
}


// makeSummaryMetrics_Lcl: Make summary metrics.
//
// Assumes:
// - 'profGbl' is the canonical CCT (with structure and with
//   canonical ids)
// - each thread-level CCT is always a subset of 'profGbl' (the
//   canonical CCT); in other words, 'profGbl' should not be pruned
//   in any way!
// - 'args' contains the correct final experiment database.
//
// FIXME: abstract between makeSummaryMetrics_Lcl() & makeThreadMetrics_Lcl()
static void
makeSummaryMetrics_Lcl(Prof::CallPath::Profile& profGbl,
                       const string& profileFile,
                       const Analysis::Args& args, uint groupId, uint groupMax,
                       vector<VMAIntervalSet*>& groupIdToGroupMetricsMap,
                       int myRank)
{
  Prof::Metric::Mgr* mMgrGbl = profGbl.metricMgr();
  Prof::CCT::Tree* cctGbl = profGbl.cct();
  Prof::CCT::ANode* cctRootGbl = cctGbl->root();

  // -------------------------------------------------------
  // read profile file
  // -------------------------------------------------------
  uint rFlags = (Prof::CallPath::Profile::RFlg_NoMetricSfx
                 | Prof::CallPath::Profile::RFlg_MakeInclExcl);
  uint rGroupId = (groupMax > 1) ? groupId : 0;

  Prof::CallPath::Profile* prof =
    Analysis::CallPath::read(profileFile, rGroupId, rFlags);

  // -------------------------------------------------------
  // merge into canonical CCT
  // -------------------------------------------------------
  int mergeTy  = Prof::CallPath::Profile::Merge_MergeMetricByName;
  int mergeFlg = (Prof::CCT::MrgFlg_AssertCCTMergeOnly);

  // Add *some* structure information to the leaves of 'prof' so that
  // it will be merged successfully with the structured canonical CCT
  // 'profGbl'.
  //
  // Background: When CCT::Stmts are merged in
  // Analysis::CallPath::coalesceStmts(CallPath::Profile), IP/LIP
  // information is not retained.  This means that when merging 'prof'
  // into 'profGbl' (using CallPath::Profile::merge()), many leaves in
  // 'prof' will not find their corresponding node in 'profGbl' unless
  // corrective measures are taken.
  prof->structure(profGbl.structure());
  Analysis::CallPath::noteStaticStructureOnLeaves(*prof);
  prof->structure(NULL);

  uint mBeg = profGbl.merge(*prof, mergeTy, mergeFlg); // [closed begin
  uint mEnd = mBeg + prof->metricMgr()->size();        //  open end)

  // -------------------------------------------------------
  // compute local incl/excl sampled metrics and update local derived metrics
  // -------------------------------------------------------

  // 1. Batch compute local sampled metrics
  VMAIntervalSet ivalsetIncl;
  VMAIntervalSet ivalsetExcl;

  for (uint mId = mBeg; mId < mEnd; ++mId) {
    Prof::Metric::ADesc* m = mMgrGbl->metric(mId);
    if (m->type() == Prof::Metric::ADesc::TyIncl) {
      ivalsetIncl.insert(VMAInterval(mId, mId + 1)); // [ )
    }
    else if (m->type() == Prof::Metric::ADesc::TyExcl) {
      ivalsetExcl.insert(VMAInterval(mId, mId + 1)); // [ )
    }
  }

  cctRootGbl->aggregateMetricsIncl(ivalsetIncl);
  cctRootGbl->aggregateMetricsExcl(ivalsetExcl);


  // 2. Batch compute local derived metrics
  const VMAIntervalSet* ivalsetDrvd = groupIdToGroupMetricsMap[groupId];
  if (ivalsetDrvd) {
    DIAG_Assert(ivalsetDrvd->size() == 1, DIAG_UnexpectedInput);
    const VMAInterval& ival = *(ivalsetDrvd->begin());
    uint mDrvdBeg = (uint)ival.beg();
    uint mDrvdEnd = (uint)ival.end();

    DIAG_MsgIf(0, "[" << myRank << "] grp " << groupId << ": [" << mDrvdBeg << ", " << mDrvdEnd << ")");
    cctRootGbl->computeMetricsIncr(*mMgrGbl, mDrvdBeg, mDrvdEnd,
                                   Prof::Metric::AExprIncr::FnAccum);
  }

  // -------------------------------------------------------
  // reinitialize metric values for next time
  // -------------------------------------------------------

  // TODO: This really should (a) come immediately after the MetricMgr
  // merge above and (b) use FnInitSrc (not 0).  However, to do this
  // we would need to (a) split the MetricMgr merge and CCT merge into
  // two; and (b) use a CCT init (which whould initialize using
  // assignment) instead of CCT::merge() (which initializes based on
  // addition against 0).
  cctRootGbl->zeroMetricsDeep(mBeg, mEnd); // cf. FnInitSrc
  
  delete prof;
}


// makeThreadMetrics_Lcl: Make thread-level metric database.
//
// Makes same assumptions as makeSummaryMetrics_Lcl but with one key
// exception: Each thread-level CCT does not have to be a subset of
// 'profGbl' (the canonical CCT); in other words, 'profGbl' may be
// pruned.
static void
makeThreadMetrics_Lcl(Prof::CallPath::Profile& profGbl,
                      const string& profileFile,
                      const Analysis::Args& args, uint groupId, uint groupMax,
                      int myRank)
{
  Prof::Metric::Mgr* mMgrGbl = profGbl.metricMgr();
  Prof::CCT::Tree* cctGbl = profGbl.cct();
  Prof::CCT::ANode* cctRootGbl = cctGbl->root();

  // -------------------------------------------------------
  // read profile file
  // -------------------------------------------------------
  uint rFlags = (Prof::CallPath::Profile::RFlg_NoMetricSfx
                 | Prof::CallPath::Profile::RFlg_MakeInclExcl);
  uint rGroupId = (groupMax > 1) ? groupId : 0;

  Prof::CallPath::Profile* prof =
    Analysis::CallPath::read(profileFile, rGroupId, rFlags);

  // -------------------------------------------------------
  // merge into canonical CCT
  // -------------------------------------------------------
  int mergeTy  = Prof::CallPath::Profile::Merge_MergeMetricByName;
  int mergeFlg = (Prof::CCT::MrgFlg_NormalizeTraceFileY
                  | Prof::CCT::MrgFlg_CCTMergeOnly);

  // Add *some* structure information to the leaves of 'prof' so that
  // it will be merged successfully with the structured canonical CCT
  // 'profGbl'.
  //
  // Background: When CCT::Stmts are merged in
  // Analysis::CallPath::coalesceStmts(CallPath::Profile), IP/LIP
  // information is not retained.  This means that when merging 'prof'
  // into 'profGbl' (using CallPath::Profile::merge()), many leaves in
  // 'prof' will not find their corresponding node in 'profGbl' unless
  // corrective measures are taken.
  prof->structure(profGbl.structure());
  Analysis::CallPath::noteStaticStructureOnLeaves(*prof);
  prof->structure(NULL);

  uint mBeg = profGbl.merge(*prof, mergeTy, mergeFlg); // [closed begin

  if (args.db_makeMetricDB) {
    uint mEnd = mBeg + prof->metricMgr()->size(); // open end)

    // -------------------------------------------------------
    // compute local incl/excl sampled metrics
    // -------------------------------------------------------

    VMAIntervalSet ivalsetIncl;
    VMAIntervalSet ivalsetExcl;
    
    for (uint mId = mBeg; mId < mEnd; ++mId) {
      Prof::Metric::ADesc* m = mMgrGbl->metric(mId);
      if (m->type() == Prof::Metric::ADesc::TyIncl) {
        ivalsetIncl.insert(VMAInterval(mId, mId + 1)); // [ )
      }
      else if (m->type() == Prof::Metric::ADesc::TyExcl) {
        ivalsetExcl.insert(VMAInterval(mId, mId + 1)); // [ )
      }
    }
    
    cctRootGbl->aggregateMetricsIncl(ivalsetIncl);
    cctRootGbl->aggregateMetricsExcl(ivalsetExcl);

    // -------------------------------------------------------
    // write local sampled metric values into database
    // -------------------------------------------------------

    string dbFnm = makeDBFileName(args.db_dir, groupId, profileFile);
    writeMetricsDB(profGbl, mBeg, mEnd, dbFnm);

    // -------------------------------------------------------
    // reinitialize metric values for next time
    // -------------------------------------------------------
    
    // TODO: see corresponding comments in makeSummaryMetrics_Lcl()
    cctRootGbl->zeroMetricsDeep(mBeg, mEnd); // cf. FnInitSrc
  }

  delete prof;
}


static string
makeDBFileName(const string& dbDir, uint groupId, const string& profileFile)
{
  string grpStr = StrUtil::toStr(groupId);
 
  string fnm_base = FileUtil::rmSuffix(FileUtil::basename(profileFile.c_str()));

  string fnm = grpStr + "." + fnm_base + "." + HPCPROF_MetricDBSfx;

  string metricDBFnm = dbDir + "/" + fnm;
  return metricDBFnm;
}


// [mBegId, mEndId)
static void
writeMetricsDB(Prof::CallPath::Profile& profGbl, uint mBegId, uint mEndId,
               const string& metricDBFnm)
{
  const Prof::CCT::Tree& cct = *(profGbl.cct());

  // -------------------------------------------------------
  // pack metrics into dense matrix
  // -------------------------------------------------------
  uint maxCCTId = cct.maxDenseId();

  ParallelAnalysis::PackedMetrics packedMetrics(maxCCTId + 1, mBegId, mEndId,
                                                mBegId, mEndId);

  ParallelAnalysis::packMetrics(profGbl, packedMetrics);

  // -------------------------------------------------------
  // write data
  // -------------------------------------------------------

  FILE* fs = hpcio_fopen_w(metricDBFnm.c_str(), 1);
  if (!fs) {
    std::string errorString;
    hpcrun_getFileErrorString(metricDBFnm, errorString);

    DIAG_EMsg("failed opening profile result file for writing " << 
              errorString << "; aborting."); 

    prof_abort(-1);
  }
  DIAG_MsgIf(0, "writeMetricsDB: " << metricDBFnm);

  uint numNodes = packedMetrics.numNodes() - 1;

  // 1. header
  hpcmetricDB_fmt_hdr_t hdr;
  hdr.numNodes = numNodes;
  hdr.numMetrics = mEndId - mBegId; // [mBegId mEndId)

  int ret;
  ret = hpcmetricDB_fmt_hdr_fwrite(&hdr, fs);
  if (ret == HPCFMT_ERR) goto badwrite;

  // 2. metric values
  //    - first row corresponds to node 1.
  //    - first column corresponds to first sampled metric.
  // cf. ParallelAnalysis::unpackMetrics: 

  for (uint nodeId = 1; nodeId < numNodes + 1; ++nodeId) {
    for (uint mId1 = 0, mId2 = mBegId; mId2 < mEndId; ++mId1, ++mId2) {
      double mval = packedMetrics.idx(nodeId, mId1);
      DIAG_MsgIf(0,  "  " << nodeId << " -> " << mval);
      ret = hpcfmt_real8_fwrite(mval, fs);
      if (ret == HPCFMT_ERR) goto badwrite;
    }
  }

  hpcio_fclose(fs);
  return;

badwrite:
  {
    std::string errorString;
    hpcrun_getFileErrorString(metricDBFnm, errorString);

    DIAG_EMsg("failed writing profile result file" << 
              errorString << "; aborting."); 
    prof_abort(-1);
  }
}


//***************************************************************************

static void
writeStructure(const Prof::Struct::Tree& structure, const char* baseNm,
               int myRank)
{
  string fnm = makeFileName(baseNm, "xml", myRank);
  std::ostream* os = IOUtil::OpenOStream(fnm.c_str());
  Prof::Struct::writeXML(*os, structure, true);
  IOUtil::CloseStream(os);
}


static void
writeProfile(const Prof::CallPath::Profile& prof, const char* baseNm,
             int myRank)
{
  // Only safe if static structure has not been added
  //string fnm_hpcrun = makeFileName(baseNm, "txt", myRank);
  //FILE* fs = hpcio_fopen_w(fnm_hpcrun.c_str(), 1);
  //Prof::CallPath::Profile::fmt_fwrite(prof, fs, 0);
  //hpcio_fclose(fs);

  string fnm_xml = makeFileName(baseNm, "xml", myRank);
  std::ostream* os = IOUtil::OpenOStream(fnm_xml.c_str());
  prof.cct()->writeXML(*os, 0, 0, Prof::CCT::Tree::OFlg_Debug);
  IOUtil::CloseStream(os);
}


static std::string
makeFileName(const char* baseNm, const char* ext, int myRank)
{
  return string(baseNm) + "-" + StrUtil::toStr(myRank) + "." + ext;
}

//****************************************************************************