Main Page | Modules | Namespace List | Class Hierarchy | Alphabetical List | Data Structures | File List | Namespace Members | Data Fields | Related Pages

Archiver.cpp

00001 #include "Archiver.h"
00002 #include "ArchivingWorkerGroup.h"
00003 #include "Sct/Archive/IOManagerArchiveFile.h" 
00004 #include "Sct/SctNames.h"
00005 #include "Sct/IpcObjectException.h"
00006 #include "Sct/UnsupportedOperationError.h"
00007 #include "Sct/StdExceptionWrapper.h"
00008 #include "Sct/ISStreamerWrapper.h"
00009 #include "is/isinfo.h"
00010 #include <pmg/pmg_initSync.h>
00011 
00012 #include "unistd.h"
00013 #include <memory>
00014 
00015 #include "TransferCommand.h"
00016 #include "IsGetCommand.h"
00017 #include "IsPutCommand.h"
00018 #include "ArchiveGetCommand.h"
00019 #include "ArchivePutCommand.h"
00020 #include "Sct/Archive/IOParamsArchive.h"
00021 #include "Sct/OmniMarshalling.h"
00022 #include "Sct/ApplicationStartupDebugTools.h"
00023 
00024 #include "ipc/core.h"
00025 
00026 using namespace std;
00027 
00028 using namespace Sct;
00029 using namespace Sct::Archive;
00030 using namespace Sct::IS;
00031 
00032 using boost::recursive_mutex;
00033 
00034 /* Obsolete:
00035 void pmgSynch(void *) {
00036     pmg_initSync();
00037 }
00038 */
00039 
00040 int main(int argc, char** argv) {
00041     using namespace SctArchiving;
00042     setExceptionHandlers(argv[0]);
00043 
00044     IPCCore::init(argc,argv);
00045     Sct::ApplicationStartupDebugTools::announceStartOfMain(argc, argv, __FILE__, __LINE__);
00046  
00047     // renice this job!
00048     nice(10);
00049     
00050     ArchiverArguments args(argc, argv);
00051     args.print(std::cout);
00052 
00053     try {
00054       Archiver& s=Archiver::initialize(args);
00055       if (!s.publish()){
00056     throw IpcObjectException("ArchivingService failed to publish", __FILE__, __LINE__);
00057       }
00058       s.go(1);
00059       pmg_initSync();
00060       s.getArchiverServer().run();
00061     } catch (Throwable& e) {
00062       e.sendToMrs(MRS_FATAL);
00063       terminate();
00064     }
00065 }
00066 
00067 namespace SctArchiving {
00068 
00069 Archiver* Archiver::archiver = 0;
00070 
00071 Archiver& Archiver::instance() {
00072   if(!archiver) throw IllegalStateError("Attempt to use uninitialised Archiver", __FILE__, __LINE__);
00073   return *archiver;
00074 }
00075 
00076 Archiver& Archiver::initialize(ArchiverArguments args){
00077   archiver = new Archiver(args);
00078   return *archiver;
00079 }
00080 
00081 Archiver::Archiver(ArchiverArguments args) : 
00082     IPCNamedObject<POA_ArchivingServiceI::ArchivingServiceInterface>(SctNames::getPartition(), args.instanceName()),
00083     nArchived(),     nRetrieved(0), 
00084     nValidated(0),   isTimeTaken(0.), 
00085     fileTimeTaken(0.), m_args(args) {
00086   
00087   auto_ptr<ISInfoReceiver> ir(new ISInfoReceiver(SctNames::getPartition()));
00088   if (!ir.get()) throw ConfigurationException("Archiver::initialize can't make infoReceiver ", __FILE__, __LINE__) ;
00089   infoReceiver = ir;
00090   workergroup = new ArchivingWorkerGroup();
00091   workergroup->reportTo(args.getISStatusName());
00092   ostringstream oss;
00093   oss << "Started with persistent directory=" << m_args.getPersistentDirectory()
00094       << " and retrieval server = " << m_args.getOutputISServer();
00095 
00096   SctNames::Mrs() << "ARCHIVING_SERVICE" << MRS_TEXT(oss.str()) 
00097           << MRS_INFORMATION << ENDM;
00098   m_archive_manager = &IOManagerArchiveFile::instance();
00099   m_retrieval_is_server=Sct::SctNames::getRetrievedDataName();
00100   m_suspend_callback=false;
00101   m_archive_manager->setCompressionLevel(m_args.getCompressionLevel());
00102 }
00103 
00104 Archiver::~Archiver() throw() {
00105   while (workergroup->busy() || workergroup->queueSize() ) {
00106     sleep(1);
00107   }
00108   workergroup->join();
00109 }
00110 
00111 void Archiver::addCommand(boost::shared_ptr<ArchivingCommand> command)const {
00112   workergroup->push(command);
00113 }
00114 
00115 IPCServer& Archiver::getArchiverServer() throw() {
00116   static IPCServer archiverServer;
00117   return archiverServer;
00118 }
00119 
00120 inline bool isControl(std::string name){
00121   return ( name.find("TestData")!=string::npos || name.find("ControlData")!=string::npos);
00122 };
00123 
00124 void Archiver::archive_callback(ISCallbackInfo *isc){
00125   try{
00126     bool over_write=isControl(isc->name());
00127     if (isc->reason() == ISInfoUpdated && !over_write ) {
00128       throw IoException(string("That is odd ... ") + isc->name() +  " has been modified! I'm not going to overwrite the data."
00129             , __FILE__, __LINE__);
00130     }
00131     if (isc->reason() != ISInfoCreated && isc->reason() != ISInfoUpdated) return;
00132     if (Archiver::instance().m_suspend_callback){
00133         return;
00134     }
00135 
00136     shared_ptr<IONameIS> name ( new IONameIS(isc->name()));
00137     shared_ptr<GetCommand> get (new IsGetCommand(name));
00138     shared_ptr<PutCommand> put (new ArchivePutCommand());
00139 
00140     if (over_write){
00141       shared_ptr<Sct::Archive::IOParamsArchive> params;
00142       params = shared_ptr<Sct::Archive::IOParamsArchive>(new IOParamsArchive());
00143       params->setOverWrite();
00144       put->setParams(params);
00145     }
00146 
00147     shared_ptr<TransferCommand> command( new TransferCommand(get, put));
00148     Archiver::instance().workergroup->push( command );
00149     
00150   } catch (Sct::Throwable& e){
00151     e.sendToMrs();
00152   }catch(std::exception& e){
00153     StdExceptionWrapper(e).sendToMrs();
00154   }
00155 }
00156 
00157 IOManagerArchive& Archiver::getIOManagerArchive() const {
00158   return *m_archive_manager;
00159 }
00160 
00161 string Archiver::internal_getRetrieveIsServer() const throw(){
00162   return  m_retrieval_is_server;
00163 }
00164 
00165   
00166 char * Archiver::getRetrieveIsServer() throw() {
00167   return copyStringToCorba(internal_getRetrieveIsServer());
00168 };
00169 
00170 void  Archiver::setRetrieveIsServer (const char * serverName){
00171   m_retrieval_is_server=serverName;
00172 }
00173 
00174 void Archiver::setPersistentDirectory(const char* newDirectory) {
00175   SctNames::setPersistentDir(newDirectory);
00176 }
00177 
00178 void Archiver::suspendCallbacks (CORBA::Boolean doSuspend){
00179    if (doSuspend) SctNames::Mrs() << MRS_TEXT("ArchivingService being suspended from callbacks") 
00180                   << "ARCHIVE_SUSPEND" << MRS_INFORMATION << ENDM;
00181    m_suspend_callback=doSuspend;
00182 }
00183 
00184 
00185 void Archiver::subscribe(const string& servername, const string& regexp, ISCallbackInfo::Callback callback){
00186   if (servername==m_args.getOutputISServer()){
00187     ostringstream oss;
00188     oss << "ArchivingService is not allowed to subscribe to [" << servername 
00189     << "] since it is publishing there and this could set up a loop";
00190     IoException(oss.str(), __FILE__, __LINE__).sendToMrs();
00191   }
00192   ISInfo::Status s=infoReceiver->subscribe(servername.c_str(), regexp.c_str(), callback);
00193   if (s!=ISInfo::Success) {
00194     ostringstream os;
00195     os <<"Archiving service could not subscribe to [" << servername
00196        << "] to retrieve [" << regexp << "]";
00197     IsException(s, os.str(), __FILE__, __LINE__).sendToMrs();
00198   } else {
00199     ostringstream oss;
00200     oss << "Archiving service subscribed to IS server [" << servername
00201     << "] to retrieve [" << regexp << "]";
00202     SctNames::Mrs() << "ARCHIVE_SUBSCRIBE" << MRS_TEXT(oss.str())
00203             << MRS_INFORMATION << ENDM;
00204   }
00205 }
00206   
00207   void Archiver::go(unsigned nWorker) throw(IsException) {
00208     const list<SctService::Arguments::Subscription> theList = m_args.getInputISServers();
00209     for (list<SctService::Arguments::Subscription>::const_iterator i=theList.begin(); 
00210      i!= theList.end(); ++i){
00211       subscribe((*i).server, (*i).regexp, archive_callback);
00212     }
00213     workergroup->go(m_args.getNWorkers());
00214     if (m_args.recoveryMode()) recover();
00215   }
00216 
00217   void Archiver::incrimentNArchived() throw(){
00218     recursive_mutex::scoped_lock scoped_lock(counterMutex);
00219     ++nArchived;
00220   }
00221   
00222   void Archiver::incrimentNRetrieved() throw(){
00223     recursive_mutex::scoped_lock scoped_lock(counterMutex);
00224     ++nRetrieved;
00225   }
00226   
00227   void Archiver::incrimentNValidated() throw(){
00228     recursive_mutex::scoped_lock scoped_lock(counterMutex);
00229     ++nValidated;
00230   }
00231   
00232   void Archiver::addISTime(double time) throw(){
00233     recursive_mutex::scoped_lock scoped_lock(counterMutex);
00234     isTimeTaken += time;
00235   }
00236 
00237   void Archiver::addFileTime(double time) throw(){
00238     recursive_mutex::scoped_lock scoped_lock(counterMutex);
00239     fileTimeTaken += time;
00240   }
00241 
00242 const char* Archiver::getStatus() const throw(){
00243   recursive_mutex::scoped_lock scoped_lock(counterMutex);
00244   ostringstream os;
00245   os << "\t=== Archiving Service ===" << endl
00246      << "nWorkers=" << workergroup->nWorkers() << endl
00247      << "\t(Busy=" << workergroup->busy() << ")" << endl
00248      << "Queue = " << workergroup->queueSize() << endl
00249      << "# Archived = " << nArchived << endl
00250      << "# Retrieved = " << nRetrieved  <<endl
00251      << "# Validated = " << nValidated << endl
00252      << " Approx average time / command / thread :" << endl 
00253      << "\tArchive = " << getAverageTime(fileTimeTaken) << endl
00254      << "\tIS/ISProxy = " << getAverageTime(isTimeTaken) << endl;
00255   os << "Compression level = " << getIOManagerArchive().getCompressionLevel() << endl;
00256   os << "IOManagerArchive Information: \n"
00257      << getIOManagerArchive().status() << endl;
00258   return os.str().c_str();
00259 }
00260 
00261   double Archiver::getAverageTime(double time) const throw(){
00262     recursive_mutex::scoped_lock scoped_lock(counterMutex);
00263     long ncommands =  nArchived + nRetrieved + nValidated;
00264     return (ncommands>0) ? time/ncommands : 0.;
00265   }
00266 
00267   char* Archiver::status(){
00268     recursive_mutex::scoped_lock scoped_lock(counterMutex);
00269     std::string status=getStatus();
00270     char *statusMessage = new char[status.size()+1];
00271     strcpy(statusMessage, status.c_str());
00272     return statusMessage;
00273   }
00274   
00275   void Archiver::archiveISName(const char* ioNameIS){
00276     try {
00277       shared_ptr<IONameIS> name( new IONameIS(ioNameIS));
00278       shared_ptr<GetCommand> get (new IsGetCommand(name));
00279       shared_ptr<PutCommand> put (new ArchivePutCommand());
00280 
00281       if (isControl(ioNameIS)){
00282     shared_ptr<Sct::Archive::IOParamsArchive> params;
00283     params = shared_ptr<Sct::Archive::IOParamsArchive>(new IOParamsArchive());
00284     params->setOverWrite();
00285     put->setParams(params);
00286       }
00287 
00288       shared_ptr<TransferCommand> command( new TransferCommand(get, put));
00289       workergroup->push( command );
00290     }catch(Throwable& e){
00291       e.sendToMrs();
00292     }
00293   }
00294 
00295   void Archiver::retrieveISName (const char* ioNameIS){
00296     try{ 
00297       IONameIS nameis(ioNameIS);
00298       IONameArchiveFile archfile( nameis.getUniqueID(), nameis.getClassName());
00300       char* blah = const_cast<char*> (archfile.getIOName().c_str());
00301       retrieveArchName (blah);
00302     }catch(Throwable& e){
00303       e.sendToMrs();
00304     }
00305   }
00306 
00307   void Archiver::retrieve(const char* runNumber, const char* scanNumber, const char* className, const char* specifier){
00308     try{ 
00309       std::ostringstream oss;
00310       oss << m_args.getPersistentDirectory() << "/" << className 
00311       << "." << runNumber << "." << scanNumber 
00312       << "." << specifier << ".gz";
00313       IONameArchiveFile arch(oss.str());
00314       char* blah = const_cast<char*>(arch.getIOName().c_str());
00315       retrieveArchName(blah);
00316     }catch(Throwable& e){
00317       e.sendToMrs();
00318     }
00319   }
00320 
00321   void Archiver::retrieveArchName (const char* archivingName){
00322     try{
00323       shared_ptr<IONameArchiveFile> name(new IONameArchiveFile(archivingName));
00324       shared_ptr<GetCommand> get (new ArchiveGetCommand(name));
00325       shared_ptr<IsPutCommand> put (new IsPutCommand());
00326       put->setServer(internal_getRetrieveIsServer());
00327       shared_ptr<TransferCommand> command( new TransferCommand(get, put));
00328       workergroup->push( command );
00329     }catch(Throwable& e){
00330       e.sendToMrs();
00331     }
00332   }
00333 
00334   
00335   CORBA::Short Archiver::busy(){
00336     return workergroup->busy();
00337   }
00338 
00339   CORBA::Short Archiver::queueLength(){
00340     return workergroup->queueSize();
00341   }
00342 
00343   CORBA::Short Archiver::getCompressionLevel () {
00344     return getIOManagerArchive().getCompressionLevel();
00345   }
00346   
00347   void Archiver::setCompressionLevel (CORBA::Short level) {
00348     getIOManagerArchive().setCompressionLevel(level);
00349   }
00350   void Archiver::recover(){
00351     SctNames::Mrs() << MRS_TEXT("Sorry - recovery not yet implimented") 
00352             << "ANALYSIS_RECOVER" << MRS_DIAGNOSTIC << ENDM;
00353   }
00354 } // end of namespace SctArchiving

Generated on Thu Feb 10 02:40:06 2005 for SCT DAQ/DCS Software - C++ by doxygen 1.3.5