structImport.module

Go to the documentation of this file.

<?php




// Include the utilities function used by the hooks and tools
include_once('./' . drupal_get_path('module', 'conStruct') . '/framework/utilities.php');


function structImport_main()
{
  global $base_url;

  drupal_set_html_head("<link type=\"text/css\" rel=\"stylesheet\" media=\"all\" href=\"" . $base_url . "/"
    . drupal_get_path("module", "structImport") . "/css/style.css\" />");

  return (conStruct_import());
}


function structImport_access($op, $node, $account)
{
  return TRUE;
}

function structImport_access_callback()
{
  return TRUE;
}


function structImport_block($op = 'list', $delta = 0, $edit = array())
{
  global $base_url;

  // The $op parameter determines what piece of information is being requested.
  switch ($op)
  {
    case 'list':

      // If $op is "list", we just need to return a list of block descriptions.
      // This is used to provide a list of possible blocks to the administrator,
      // end users will not see these descriptions.
      $blocks[0]['info'] = t('conStruct Import Tool');
      return $blocks;

    case 'configure':
      // If $op is "configure", we need to provide the administrator with a
      // configuration form. The $delta parameter tells us which block is being
      // configured. In this example, we'll allow the administrator to customize
      // the text of the first block.
      $form = array();
      if ($delta == 0)
      {
        // All we need to provide is a text field, Drupal will take care of
        // the other block configuration options and the save button.
        $form['block_example_string'] = array(
          '#type' => 'textfield',
          '#title' => t('Block contents'),
          '#size' => 100,
          '#description' => t('This string will appear in the conStruct Tools menu'),
          '#default_value' => variable_get('conStruct_string', t(''))
        );
      }

      return $form;

    case 'save':

      // If $op is "save", we need to save settings from the configuration form.
      // Since the first block is the only one that allows configuration, we
      // need to check $delta to make sure we only save it.
      if ($delta == 0)
      {
        // Have Drupal save the string to the database.
        variable_set('structImport_string', $edit['structImport_string']);
      }
      return;

    case 'view':

    default:

      // If $op is "view", then we need to generate the block for display
      // purposes. The $delta parameter tells us which block is being requested.
      switch ($delta)
      {
        case 0:
          // The subject is displayed at the top of the block. Note that it
          // should be passed through t() for translation.
          $block['subject'] = t('conStruct Import Tool');
          // The content of the block is typically generated by calling a custom
          // function.
          $block['content'] = structImport_contents(1);
          break;
      }
      return $block;
  }
}

function structImport_contents($which_block)
{
  global $base_url;

  if ($which_block == 1)
  {
    // Get the base path of the Drupal installation folder on the file system of the server
    $menuItems = "";

    if (user_access('access conStruct import'))
    {
      $localPath = $base_url . "/" . drupal_get_path("module", "structImport") . "/imgs/";

      // For all users
      $menuItems .= "<img src=\"" . $localPath . "view_detailed.png\" style=\"padding-right:5px;\" alt=\"\" /><a href=\""
        . $base_url . "/conStruct/import/\">Import</a><br />";
    }

    return variable_get('structImport_menu', t($menuItems));
  }
}

function structImport_menu()
{
  $items = array();

  // Registration of the conStruct module settings page path
  /*     
      $items['admin/settings/structImport'] = array(  'title' => '',
                                          'description' => '',
                                          'page callback' => 'drupal_get_form',
                                          'page arguments' => array('structImport_admin'),
                                          'access arguments' => array('access administration pages'),
                                          'type' => MENU_NORMAL_ITEM);
  */
  // Registration of the main conStruct module page path
  $items['conStruct/import'] = array(
    'page callback' => 'structImport_main',
    'access callback' => 'structImport_access_callback',
    'type' => MENU_CALLBACK
  );

  return $items;
}


function structImport_perm()
{
  return array(
    'access conStruct import',
    'administer conStruct import'
  );
}


function conStruct_import()
{
  // Make sure the user has access to this tool
  if (!user_access('access conStruct import'))
  {
    return "<h2>" . t("You don't have access to this tool, sorry.") . "</h2>";
  }
  else
  {
    // Display errors when in debug mode
    /*
    if (!isset($_GET['debug']) || $_GET['debug'] != 1)
    {
      ini_set("error_reporting", 0);
    }
    */
    
    // always display errors for this module
    ini_set("display_errors", "On"); 
    error_reporting(E_ALL);    

    $html = "";

    // Check if a file has been uploaded
    if (isset($_FILES) && count($_FILES) > 0)
    {
      // Check if the upload of the file is sucessful
      if ($_FILES['file']['error'] == UPLOAD_ERR_OK)
      {
        // Once we start the importation process, we have to make sure that even if the server
        // loose the connection with the user the process will still finish.
        ignore_user_abort(true);

        // However, maybe there is an issue with the server handling that file tht lead to some kind of infinite or near
        // infinite loop; so we have to limit the execution time of this procedure to 45 mins.
        set_time_limit(2700);

        // upload ok
        include_once('./' . drupal_get_path('module', 'conStruct') . '/framework/WebServiceQuerier.php');
        include_once('./' . drupal_get_path('module', 'conStruct') . '/framework/ProcessorXML.php');

        $contentType = "";

        if (isset($_POST["ctype"]))
        {
          $contentType = $_POST["ctype"];
        }

        $datasetName = "Unnamed dataset";

        if (isset($_POST["dname"]))
        {
          $datasetName = $_POST["dname"];
        }

        $datasetDescription = "Unnamed dataset";

        if (isset($_POST["ddescription"]))
        {
          $datasetDescription = $_POST["ddescription"];
        }

        $wsfAddress = "http://localhost/ws/";

        if (isset($_POST["structwsf"]))
        {
          $wsfAddress = $_POST["structwsf"];
        }

        $rdfDocument = "";

        if (!move_uploaded_file($_FILES['userfile']['tmp_name'], "/tmp/" . $_FILES['userfile']['name']))
        {
          return "";
        }

        // Read file form the server
        $file = fopen("/tmp/" . $_FILES['userfile']['name'], "r");

        while (!feof($file))
        {
          $fileContent .= fread($file, 8192);
        }

        fclose($file);

        unlink("/tmp/" . $_FILES['userfile']['name']);

        global $user;
        
        // Create a new dataset where to index the imported data.
        // We create a new dataset by creating a OG group which mean
        // creating a new drupal Node which is a Group Node...

        $node = array('type' => 'dataset');

        // Create a new Dataset where to index the imported dataset
        $form_state = array();
        module_load_include('inc', 'node', 'node.pages');
        $node = array('type' => 'dataset');
        $form_state['values']['title'] = $datasetName;
        $form_state['values']['og_description'] = $datasetDescription;
        $form_state['values']['name'] = $user->name;
        $form_state['values']['field_wsf'][0]['value'] =
          substr(get_domain($wsfAddress), 7, strlen(get_domain($wsfAddress)) - 7);
          
        $form_state['values']['op'] = t('Save');

        drupal_execute('dataset_node_form', $form_state, (object)$node);

        $datasetDomain = variable_get("conStruct_UrisDomain", "localhost");

        $userIP = variable_get("conStruct_AccessUser" . $user->uid, "");

        if ($userIP == "")
        {
          // If there is no IP for that user, we overload the IP of the node with "::"
          $userIP = "self::$user->uid";
        }

        $dataset = urlencode("http://" . $datasetDomain . "/wsf/datasets/" . $form_state["nid"] . "/");

        // Handle conversion of the dataset if needed
        switch ($contentType)
        {
          case "bibjson":

          case "irjson":
            $wsq = new WebServiceQuerier($wsfAddress . "converter/irjson/", "post", "application/rdf+xml",
              "document=" . urlencode($fileContent) .
            "&docmime=" . urlencode("application/iron+json") .
            "&registered_ip=" . urlencode($userIP));
            if ($wsq->getStatus() != 200)
            {
              // In case of error, we have to delete the node we created above.
              node_delete($form_state["nid"]);
              
              $wsq->displayError();

              return ("");
            }

            $rdfDocument = $wsq->getResultset();

            $contentType = "application/rdf+xml";

            unset($wsq);
          break;

          case "common":
            $wsq = new WebServiceQuerier($wsfAddress . "converter/common/", "post", "application/rdf+n3",
              "document=" . urlencode($fileContent) .
              "&docmime=" . urlencode("application/iron+csv") .
              "&registered_ip=" . urlencode($userIP));
            
            if ($wsq->getStatus() != 200)
            {
              // In case of error, we have to delete the node we created above.
              node_delete($form_state["nid"]);
              
              $wsq->displayError();

              return ("");
            }

            $rdfDocument = $wsq->getResultset();

            $contentType = "application/rdf+n3";

            unset($wsq);

          break;
          
          case "bibtex":
            $wsq = new WebServiceQuerier($wsfAddress . "converter/bibtex/", "post", "application/rdf+xml",
              "document=" . urlencode($fileContent) .
              "&docmime=" . urlencode("application/x-bibtex") .
              "&registered_ip=" . urlencode($userIP) .
              "&base_uri=" . $dataset . "resource/");
            if ($wsq->getStatus() != 200)
            {
              // In case of error, we have to delete the node we created above.
              node_delete($form_state["nid"]);
              
              $wsq->displayError();

              return ("");
            }

            $rdfDocument = $wsq->getResultset();
            $contentType = "application/rdf+xml";

            unset($wsq);
            break;

          case "tsv":
            $wsq = new WebServiceQuerier($wsfAddress . "converter/tsv/", "post", "application/rdf+xml",
              "document=" . urlencode($fileContent) .
              "&docmime=" . urlencode("text/csv") .
              "&delimiter=\t" .
              "&registered_ip=" . urlencode($userIP) .
              "&base_uri=" . $dataset . "resource/");
            if ($wsq->getStatus() != 200)
            {
              // In case of error, we have to delete the node we created above.
              node_delete($form_state["nid"]);
              
              $wsq->displayError();

              return ("");
            }

            $rdfDocument = $wsq->getResultset();
            $contentType = "application/rdf+xml";

            unset($wsq);
            break;

          case "csv":
            $wsq = new WebServiceQuerier($wsfAddress . "converter/tsv/", "post", "application/rdf+xml",
              "document=" . urlencode($fileContent) .
              "&docmime=" . urlencode("text/csv") .
              "&delimiter=" . urlencode(","), "&registered_ip=" . urlencode($userIP) .
              "&base_uri=" . $dataset . "resource/");
            if ($wsq->getStatus() != 200)
            {
              // In case of error, we have to delete the node we created above.
              node_delete($form_state["nid"]);

              $wsq->displayError();

              return ("");
            }

            $rdfDocument = $wsq->getResultset();
            $contentType = "application/rdf+xml";

            unset($wsq);
            break;

          case "rdf_xml":
            $rdfDocument = $fileContent;
            $contentType = "application/rdf+xml";
            break;

          case "rdf_n3":
            $rdfDocument = $fileContent;
            $contentType = "application/rdf+n3";
            break;

          default:
          
            // In case of error, we have to delete the node we created above.
            node_delete($form_state["nid"]);
          
            drupal_set_message(t("Unknown content type"), "error", TRUE);

            return "";
            break;
        }

        // Now check if some of the attributes/types imported  are not currently defined in the
        // internal ontological structure.
        if(isset($_POST["checkAT"]) && $_POST["checkAT"] == "on")
        {
          include_once('./' . drupal_get_path('module', 'conStruct') . '/framework/arc2/ARC2.php');
          
          $parser = ARC2::getRDFParser();
          $parser->parse("", $rdfDocument);
          
          if($contentType == "application/rdf+xml")
          {
            $rdfxmlSerializer = ARC2::getRDFXMLSerializer();
          }
          
          if($contentType == "application/rdf+n3")
          {
            $rdfxmlSerializer = ARC2::getNTriplesSerializer();
          }

          $resourceIndex = $parser->getSimpleIndex(0);

          $usedProperties = array();
          
          $usedTypes = array();
                       
          // If we can read the generated rdf document with ARC2, we try to figure out if some
          // types/properties have been used which are not in the ontological structure.
          if(count($parser->getErrors()) <= 0 && $rdfxmlSerializer)
          {
            foreach($resourceIndex as $resource => $description)
            {
              foreach($description as $predicate => $values)
              {
                if($predicate == "http://www.w3.org/1999/02/22-rdf-syntax-ns#type")
                {
                  foreach($values as $value)
                  {
                    if($value["type"] == "uri")
                    {
                      if(array_search($value["value"], $usedTypes) === FALSE)
                      {
                        array_push($usedTypes, $value["value"]);
                      }
                    }
                  }
                }
                else
                {
                  if(array_search($predicate, $usedProperties) === FALSE)
                  {
                    array_push($usedProperties, $predicate);
                  }
                }
              }

              if($break)
              {
                break;
              }
            }        
          }         
          unset($parser);
        }
      
        // Import dataset by using the CrudCreate web service
        
        $wsq = new WebServiceQuerier($wsfAddress . "crud/create/", "post", "text/xml",
          "document=" . urlencode($rdfDocument) .
          "&dataset=" . $dataset .
          "&registered_ip=" . urlencode($userIP) .
          "&mime=" . urlencode($contentType) .
          "&mode=full");         
          
        if($wsq->getStatus() != 200)
        {
          // In case of error, we have to delete the node we created above.
          node_delete($form_state["nid"]);
            
          // The size of the converted document is too big.
          if($wsq->error->id == "WSF-600")
          {
            // In case of error, we have to delete the node we created above.
            node_delete($form_state["nid"]);          
            
            $uploadMaxFileSize = ini_get("post_max_size");
            $uploadMaxFileSize = str_replace("M", "000000", $uploadMaxFileSize);            
            
            // The converted dataset is too big to be processed by this tool
            drupal_set_message(t("The converted dataset file is too big to be processed by
                                 this import tool. You will have to split the file in multiple
                                 slices that you will import using this tool, and then append
                                 using the Append tool. Or you will have to create a new
                                 procedure that can handle files of that size. The size of the
                                 converted file is @sizeConverted bytes, and the autorized
                                 file size is @sizeAuthorized bytes", 
                                 array("@sizeConverted" => number_format(strlen($rdfDocument), 0, " ", " "),
                                       "@sizeAuthorized" => number_format($uploadMaxFileSize, 0, " ", " "))), 
                                       "error", TRUE);     
          }          
          else
          {          
            $wsq->displayError();
          }

          return ("");
        }

        unset($wsq);

        $undefinedPredicates = array();
        $undefinedTypes = array();
        
        if(isset($_POST["checkAT"]) && $_POST["checkAT"] == "on")
        {        
          // Now check to make sure that all the predicates and types are in the ontological structure.
          include_once('./' . drupal_get_path('module', 'conStruct') . '/framework/ClassHierarchy.php');
          include_once('./' . drupal_get_path('module', 'conStruct') . '/framework/PropertyHierarchy.php');
          include_once(drupal_get_path('module', 'structBrowse') . '/settings.php');

          $userIP = variable_get("conStruct_AccessUser" . $user->uid, "");
          $filename = './' . drupal_get_path('module', 'conStruct') . '/framework/ontologies/classHierarchySerialized.srz';
          $file = fopen($filename, "r");
          $classHierarchy = fread($file, filesize($filename));
          $classHierarchy = unserialize($classHierarchy);
          fclose($file);

          $filename = './' . drupal_get_path('module', 'conStruct') . '/framework/ontologies/propertyHierarchySerialized.srz';
          $file = fopen($filename, "r");
          $propertyHierarchy = fread($file, filesize($filename));
          $propertyHierarchy = unserialize($propertyHierarchy);
          fclose($file);
          
          foreach($usedProperties as $usedPredicate)
          {
            $found = FALSE;
            foreach($propertyHierarchy->properties as $property)
            {
              if($property->name == $usedPredicate)
              {
                $found = TRUE;
                break;
              }
            }
            
            if($found === FALSE)
            {
              array_push($undefinedPredicates, $usedPredicate);
            }
          }
          
          foreach($usedTypes as $type)
          {
            $found = FALSE;
            foreach($classHierarchy->classes as $class)
            {
              if($class->name == $type)
              {
                $found = TRUE;
                break;
              }
            }
            
            if($found === FALSE)
            {
              array_push($undefinedTypes, $type);
            }
          }    
        }
        else
        {
          drupal_set_message(t("Checking of un-handled attributes and types has been disabled by the importer.
                                Make sure that all the types and attributes used to describe the imported
                                dataset are defined in the ontological structure of this node. Otherwise
                                unwelcome behaviors could happen."), "warning", TRUE);     
                                
          return("<form><input type=\"button\" value=\"Browse Imported Dataset\" onClick=\"parent.location='../browse/?browse=1&page=0&attribute=all&type=all&dataset=$dataset'\" /></form>");
        }    
        
        if(count($undefinedPredicates) == 0 && count($undefinedTypes) == 0)
        {
          // Redirect to its browse page
          header("Location: ../browse/?browse=1&page=0&attribute=all&type=all&dataset=" . $dataset);
        }
        else
        {
          // We display the missing predicates & classes notices with a button
          
          $pErrors = "";
          
          if(count($undefinedPredicates) > 0)
          {
            $pErrors = "<h2>Undefined Attributes</h2><ol>";
            foreach($undefinedPredicates as $p)
            {
              $pErrors .= "<li>$p</li>";
            }
            $pErrors .= "</ol>";
          }
          
          $errorMsg = "Warning: some types and attributes were used in the file that you were importing that are not
                       currently defined in the ontological structure of this system. Everything will works
                       fine, but some unexpected display behaviors could occurs for some usecase. We
                       suggest to take these attributes and types in node to define them in the ontological
                       structure in the future.";

          drupal_set_message(t("@warning", array("@warning" => $errorMsg)), "warning", TRUE);                       

          foreach($undefinedPredicates as $p)
          {
            drupal_set_message(t("Undefined attributes: @warning", array("@warning" => $p)), "warning", TRUE);     
          }

          foreach($undefinedTypes as $t)
          {
            drupal_set_message(t("Undefined types: @warning", array("@warning" => $t)), "warning", TRUE);     
          }
                  
          return("<form><input type=\"button\" value=\"Browse Imported Dataset\" onClick=\"parent.location='../browse/?browse=1&page=0&attribute=all&type=all&dataset=$dataset'\" /></form>");
        }
      }
      else
      {
        drupal_set_message(t("File upload failed with error: @error",
          array("@error" => file_upload_error_message($_FILES['file']['error']))), "error", TRUE);
          
        return("");          
      }
    }
    else
    {
      $html .= "<h2>Import Dataset</h2>";

      // Get all available structWSF instances available to this conStruct node

      /*
      $sidRegistry = variable_get("SID-Registry", array());

      $sidHtml = "";

      foreach ($sidRegistry as $s => $wsfs)
      {
        $sidHtml .= "<option value=\"" . $wsfs[0] . "\" /> " . $wsfs[0];
      }
      */
      
      $wsfRegistry = variable_get("WSF-Registry", array());

      $sidHtml = "";

      foreach ($wsfRegistry as $s => $wsfs)
      {
        $sidHtml .= "<option value=\"" . $wsfs . "\" /> " . $wsfs;
      }      

      // If no SID are available, lets test if there is a local structWSF instance available
      if ($sidHtml == "")
      {
        $ch = curl_init();

        curl_setopt($ch, CURLOPT_HEADER, 0);
        curl_setopt($ch, CURLOPT_URL, "http://localhost/ws/index.php");
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);

        $data = curl_exec($ch);

        if (curl_errno($ch))
        {
          curl_close($ch);

          // no local SID are existing, so we send an error to the user.
          drupal_set_message(t("No structWSF instance are available for this conStruct node. Please" .
            "contact the system administrator to link this conStruct instance" .
            "to a structWSF instance."), "error", TRUE);

          return ("");
        }
        else
        {
          $sidHtml .= "<option value=\"http://localhost/ws/\" /> http://localhost/ws/";
        }

        curl_close($ch);
      }

      $html
        .= "   <form enctype=\"multipart/form-data\" action=\"\" method=\"POST\">
            
                            <!--<input type=\"hidden\" name=\"MAX_FILE_SIZE\" value=\"30000\" />-->
            
                           <table width=\"100%\" border=\"0\">
                               <tr><td>&nbsp;</td><td>&nbsp;</td></tr>
                               <tr>
                                   <td>
                                    Dataset file to import:
                                   </td>
                                   <td>
                                    <input name=\"userfile\" type=\"file\" size=\"61\" />
                                   </td>
                               </tr>
                               <tr>
                                   <td>
                                    Content type:
                                   </td>
                                   <td>
                                    <select name=\"ctype\" size=\"1\" style=\"width: 400px;\">
                                       <!--<option value=\"bibjson\" /> BibJSON-->
                          <option value=\"common\" /> commON (csv)
                          <option value=\"irjson\" /> irJSON
                                       <!--<option value=\"bibtex\" /> BibTex-->
                                       <option value=\"tsv\" /> N-Triples/TSV
                                       <option value=\"csv\" /> N-Triples/CSV
                                       <option value=\"rdf_xml\" /> RDF+XML
                                       <option value=\"rdf_n3\" /> RDF+N3
                                    </select>
                                   </td>
                               </tr>
                               <tr>
                                   <td>
                                    Dataset name:
                                   </td>
                                   <td>
                                    <input name=\"dname\" type=\"text\" size=\"64\" maxlength=\"256\" />
                                   </td>
                               </tr>
                               <tr>
                                   <td>
                                    Dataset description:
                                   </td>
                                   <td>
                                    <input name=\"ddescription\" type=\"text\" size=\"64\" maxlength=\"256\" />
                                   </td>
                               </tr>
                      <tr>
                          <td>
                        Save dataset on this network:
                          </td>
                          <td>
                        <select name=\"structwsf\" size=\"1\" style=\"width: 400px;\">
                          $sidHtml
                        </select>
                          </td>
                      </tr>
                      <tr>
                          <td style=\"padding-top: 15px;\">
                        Check attributes and types existance: <br>
                        <b>Time consuming</b>
                          </td>
                          <td style=\"padding-top: 15px;\">
                          <input type=\"checkbox\" name=\"checkAT\" checked> Check for missing attributes and types
                          in the imported dataset. <br />This should be unchecked for big datasets because of
                          the resources needed to make this check.
                          </td>
                      </tr>
                              <tr><td>&nbsp;</td><td>&nbsp;</td></tr>
                               <tr>
                                   <td colspan=\"2\" align=\"center\">
                                    <input type=\"submit\" value=\"Import Dataset\" />
                                   </td>
                               </tr>
                           </table>
                        </form>  
                     ";
    }

    return ($html);
  }
}

function file_upload_error_message($error_code)
{
  switch ($error_code)
  {
    case UPLOAD_ERR_INI_SIZE:

      {
        return 'File size is too big. The maximum file size is 8mb.';
      }

    case UPLOAD_ERR_FORM_SIZE:

      {
        return 'File size is too big. The maximum file size is 8mb.';
      }

    case UPLOAD_ERR_PARTIAL:

      {
        return 'The uploaded file was only partially uploaded';
      }

    case UPLOAD_ERR_NO_FILE:

      {
        return 'No file was uploaded';
      }

    case UPLOAD_ERR_NO_TMP_DIR:

      {
        return 'Missing a temporary folder';
      }

    case UPLOAD_ERR_CANT_WRITE:

      {
        return 'Failed to write file to disk';
      }

    case UPLOAD_ERR_EXTENSION:

      {
        return 'File upload stopped by extension';
      }

    default:
      return 'Unknown upload error';
  }
}
?>