Update to ARC 4.1.0
[gp-arc-client-c.git] / ext / arc_get.cpp
index 415c2e9..1f22a2c 100644 (file)
@@ -3,7 +3,6 @@
  *  arcclientc
  *
  *  Created by Tamas Jung on 10/25/10.
- *  Copyright 2010 Apple. All rights reserved.
  *
  */
 
 
 // -*- indent-tabs-mode: nil -*-
 
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
+
 #include "logger.h"
 #include <iostream>
 #include <list>
 #include <string>
+#include <sys/stat.h>
 #include "arc_libs.h"
 
+#include "utils.h"
 
 int arc_get(int argc, char **argv) {
        
-       
+  Arc::ArcLocation::Init(argv[0]);
 
-       
-       Arc::OptionParser options(istring("[job ...]"),
-                                                         istring("The arcget command is used for "
-                                                                         "retrieving the results from a job."),
-                                                         istring("Argument to -c has the format "
-                                                                         "Flavour:URL e.g.\n"
-                                                                         "ARC0:ldap://grid.tsl.uu.se:2135/"
-                                                                         "nordugrid-cluster-name=grid.tsl.uu.se,"
-                                                                         "Mds-Vo-name=local,o=grid"));
-       
-       
-       std::string joblist;
-       options.AddOption('j', "joblist",
-                                         istring("file containing a list of jobs"),
-                                         istring("filename"),
-                                         joblist);
-       
-       std::list<std::string> clusters;
-       options.AddOption('c', "cluster",
-                                         istring("explicitly select or reject a specific resource"),
-                                         istring("[-]name"),
-                                         clusters);
-       
-       std::list<std::string> status;
-       options.AddOption('s', "status",
-                                         istring("only select jobs whose status is statusstr"),
-                                         istring("statusstr"),
-                                         status);
-       
-       std::string downloaddir;
-       options.AddOption('D', "dir",
-                                         istring("download directory (the job directory will"
-                                                         " be created in this directory)"),
-                                         istring("dirname"),
-                                         downloaddir);
-       
-       bool keep = false;
-       options.AddOption('k', "keep",
-                                         istring("keep the files on the server (do not clean)"),
-                                         keep);
-       
-       int timeout = -1;
-       options.AddOption('t', "timeout", istring("timeout in seconds (default 20)"),
-                                         istring("seconds"), timeout);
-       
-       std::string conffile;
-       options.AddOption('z', "conffile",
-                                         istring("configuration file (default ~/.arc/client.conf)"),
-                                         istring("filename"), conffile);
-       
-       
-       
-       std::list<std::string> jobs = options.Parse(argc, argv);
-       
+  ClientOptions opt(ClientOptions::CO_GET,
+                    istring("[job ...]"),
+                    istring("The arcget command is used for "
+                            "retrieving the results from a job."));
 
-               
-               
-       Arc::UserConfig usercfg(conffile, joblist);
-       if (!usercfg) {
-               logger.msg(Arc::ERROR, "Failed configuration initialization");
-               return 1;
-       }
-               
-       if (timeout > 0)
-       usercfg.Timeout(timeout);
-               
-       if (jobs.empty() && clusters.empty()) {
-               logger.msg(Arc::ERROR, "No jobs given");
-               return 1;
-       }
-       
-       if (!jobs.empty())
-               usercfg.ClearSelectedServices();
-       
-       if (!clusters.empty()) {
-               usercfg.ClearSelectedServices();
-               usercfg.AddServices(clusters, Arc::COMPUTING);
-       }
-       
-       Arc::JobSupervisor jobmaster(usercfg, jobs);
-       if (!jobmaster.JobsFound()) {
-               logger.msg(Arc::ERROR, "No jobs");
-               return 1;
-       }
-       std::list<Arc::JobController*> jobcont = jobmaster.GetJobControllers();
-       
-       // If the user specified a joblist on the command line joblist equals
-       // usercfg.JobListFile(). If not use the default, ie. usercfg.JobListFile().
-       if (jobcont.empty()) {
-               logger.msg(Arc::ERROR, "No job controller plugins loaded");
-               return 1;
-       }
-  int result = 0;
-       for (std::list<Arc::JobController*>::iterator it = jobcont.begin();
-                it != jobcont.end(); it++){
-               if (!(*it)->Get(status, downloaddir, keep)){
-                       result = 1;
-               }
-       }
-       
-       return result;
+  std::list<std::string> jobidentifiers = opt.Parse(argc, argv);
+
+  if (opt.showversion) {
+    std::cout << Arc::IString("%s version %s", "arcget", "4.1.0")
+              << std::endl;
+    return 0;
+  }
+
+  // If debug is specified as argument, it should be set before loading the configuration.
+  if (!opt.debug.empty())
+    Arc::Logger::getRootLogger().setThreshold(Arc::istring_to_level(opt.debug));
+
+  logger.msg(Arc::VERBOSE, "Running command: %s", opt.GetCommandWithArguments());
+
+  if (opt.show_plugins) {
+    std::list<std::string> types;
+    types.push_back("HED:JobControllerPlugin");
+    showplugins("arcget", types, logger);
+    return 0;
+  }
+
+  Arc::UserConfig usercfg(opt.conffile, opt.joblist);
+  if (!usercfg) {
+    logger.msg(Arc::ERROR, "Failed configuration initialization");
+    return 1;
+  }
+
+  if (!checkproxy(usercfg)) {
+    return 1;
+  }
+
+  if (opt.debug.empty() && !usercfg.Verbosity().empty())
+    Arc::Logger::getRootLogger().setThreshold(Arc::istring_to_level(usercfg.Verbosity()));
+
+  if (opt.downloaddir.empty()) {
+    if (!usercfg.JobDownloadDirectory().empty()) {
+      opt.downloaddir = usercfg.JobDownloadDirectory();
+      logger.msg(Arc::INFO, "Job download directory from user configuration file: %s ", opt.downloaddir);
+    }
+    else {
+      logger.msg(Arc::INFO, "Job download directory will be created in present working directory. ");
+    }
+  }
+  else {
+    logger.msg(Arc::INFO, "Job download directory: %s ", opt.downloaddir);
+  }
+
+  for (std::list<std::string>::const_iterator it = opt.jobidinfiles.begin(); it != opt.jobidinfiles.end(); it++) {
+    if (!Arc::Job::ReadJobIDsFromFile(*it, jobidentifiers)) {
+      logger.msg(Arc::WARNING, "Cannot read specified jobid file: %s", *it);
+    }
+  }
+
+  if (opt.timeout > 0)
+    usercfg.Timeout(opt.timeout);
+
+  if ((!opt.joblist.empty() || !opt.status.empty()) && jobidentifiers.empty() && opt.clusters.empty())
+    opt.all = true;
+
+  if (jobidentifiers.empty() && opt.clusters.empty() && !opt.all) {
+    logger.msg(Arc::ERROR, "No jobs given");
+    return 1;
+  }
+
+  std::list<std::string> selectedURLs;
+  if (!opt.clusters.empty()) {
+    selectedURLs = getSelectedURLsFromUserConfigAndCommandLine(usercfg, opt.clusters);
+  }
+  std::list<std::string> rejectManagementURLs = getRejectManagementURLsFromUserConfigAndCommandLine(usercfg, opt.rejectmanagement);
+
+  std::list<Arc::Job> jobs;
+  Arc::JobInformationStorage *jobstore = createJobInformationStorage(usercfg);
+  if (jobstore != NULL && !jobstore->IsStorageExisting()) {
+    logger.msg(Arc::ERROR, "Job list file (%s) doesn't exist", usercfg.JobListFile());
+    delete jobstore;
+    return 1;
+  }
+  if (jobstore == NULL ||
+      ( opt.all && !jobstore->ReadAll(jobs, rejectManagementURLs)) ||
+      (!opt.all && !jobstore->Read(jobs, jobidentifiers, selectedURLs, rejectManagementURLs))) {
+    logger.msg(Arc::ERROR, "Unable to read job information from file (%s)", usercfg.JobListFile());
+    delete jobstore;
+    return 1;
+  }
+
+  if (!opt.all) {
+    for (std::list<std::string>::const_iterator itJIdentifier = jobidentifiers.begin();
+         itJIdentifier != jobidentifiers.end(); ++itJIdentifier) {
+      std::cout << Arc::IString("Warning: Job not found in job list: %s", *itJIdentifier) << std::endl;
+    }
+  }
+
+  Arc::JobSupervisor jobmaster(usercfg, jobs);
+
+  jobmaster.Update();
+  jobmaster.SelectValid();
+  if (!opt.status.empty()) {
+    jobmaster.SelectByStatus(opt.status);
+  }
+
+  if (jobmaster.GetSelectedJobs().empty()) {
+    std::cout << Arc::IString("No jobs") << std::endl;
+    delete jobstore;
+    return 1;
+  }
+
+  if(!opt.downloaddir.empty()) {
+    Arc::URL dirpath(opt.downloaddir);
+    if(dirpath.Protocol() == "file") {
+      if(!Arc::DirCreate(dirpath.Path(),S_IRWXU,true)) {
+        std::string errstr = Arc::StrError();
+        logger.msg(Arc::ERROR, "Unable to create directory for storing results (%s) - %s", dirpath.Path(), errstr);
+        return 1;
+      }
+    }
+  }
+  std::list<std::string> downloaddirectories;
+  int retval = (int)!jobmaster.Retrieve(opt.downloaddir, opt.usejobname, opt.forcedownload, downloaddirectories);
+
+  for (std::list<std::string>::const_iterator it = downloaddirectories.begin();
+       it != downloaddirectories.end(); ++it) {
+    std::cout << Arc::IString("Results stored at: %s", *it) << std::endl;
+  }
+
+  unsigned int processed_num = jobmaster.GetIDsProcessed().size();
+  unsigned int retrieved_num = downloaddirectories.size();
+  unsigned int cleaned_num = 0;
+
+  if (!opt.keep) {
+    std::list<std::string> retrieved = jobmaster.GetIDsProcessed();
+    // No need to clean selection because retrieved is subset of selected
+    jobmaster.SelectByID(retrieved);
+    if(!jobmaster.Clean()) {
+      std::cout << Arc::IString("Warning: Some jobs were not removed from server") << std::endl;
+      std::cout << Arc::IString("         Use arclean to remove retrieved jobs from job list", usercfg.JobListFile()) << std::endl;
+      retval = 1;
+    }
+    cleaned_num = jobmaster.GetIDsProcessed().size();
+
+    if (!jobstore->Remove(jobmaster.GetIDsProcessed())) {
+      std::cout << Arc::IString("Warning: Failed removing jobs from file (%s)", usercfg.JobListFile()) << std::endl;
+      std::cout << Arc::IString("         Use arclean to remove retrieved jobs from job list", usercfg.JobListFile()) << std::endl;
+      retval = 1;
+    }
+
+    std::cout << Arc::IString("Jobs processed: %d, successfully retrieved: %d, successfully cleaned: %d", processed_num, retrieved_num, cleaned_num) << std::endl;
+
+  } else {
+
+    std::cout << Arc::IString("Jobs processed: %d, successfully retrieved: %d", processed_num, retrieved_num) << std::endl;
+
+  }
+  delete jobstore;
+
+  return retval;       
 }