libdap  Updated for version 3.19.1
libdap4 is an implementation of OPeNDAP's DAP protocol.
D4Connect.cc
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of libdap, A C++ implementation of the OPeNDAP Data
4 // Access Protocol.
5 
6 // Copyright (c) 2002,2003 OPeNDAP, Inc.
7 // Author: James Gallagher <jgallagher@opendap.org>
8 // Dan Holloway <dholloway@gso.uri.edu>
9 // Reza Nekovei <reza@intcomm.net>
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
26 
27 // (c) COPYRIGHT URI/MIT 1994-2002
28 // Please read the full copyright statement in the file COPYRIGHT_URI.
29 //
30 // Authors:
31 // jhrg,jimg James Gallagher <jgallagher@gso.uri.edu>
32 // dan Dan Holloway <dholloway@gso.uri.edu>
33 // reza Reza Nekovei <reza@intcomm.net>
34 
35 #include "config.h"
36 // #define DODS_DEBUG 1
37 
38 
39 
40 #include <cassert>
41 #include <cstring>
42 #include <sstream>
43 
44 #include "D4Connect.h"
45 #include "HTTPConnect.h"
46 #include "Response.h"
47 #include "DMR.h"
48 #include "D4Group.h"
49 
50 #include "D4ParserSax2.h"
51 #include "chunked_stream.h"
52 #include "chunked_istream.h"
53 #include "D4StreamUnMarshaller.h"
54 
55 #include "escaping.h"
56 #include "mime_util.h"
57 #include "debug.h"
58 
59 
60 
61 using namespace std;
62 
63 namespace libdap {
64 
67 void D4Connect::process_dmr(DMR &dmr, Response &rs)
68 {
69  DBG(cerr << "Entering D4Connect::process_dmr" << endl);
70 
71  dmr.set_dap_version(rs.get_protocol());
72 
73  DBG(cerr << "Entering process_data. Response.getVersion() = " << rs.get_version() << endl);
74  switch (rs.get_type()) {
75  case dap4_error: {
76 #if 0
77  Error e;
78  if (!e.parse(rs.get_stream()))
79  throw InternalErr(__FILE__, __LINE__, "Could not parse the Error object returned by the server!");
80  throw e;
81 #endif
82  throw InternalErr(__FILE__, __LINE__, "DAP4 errors not processed yet: FIXME!");
83  }
84 
85  case web_error:
86  // Web errors (those reported in the return document's MIME header)
87  // are processed by the WWW library.
88  throw InternalErr(__FILE__, __LINE__,
89  "An error was reported by the remote httpd; this should have been processed by HTTPConnect..");
90 
91  case dap4_dmr: {
92  // parse the DMR
93  try {
94  D4ParserSax2 parser;
95  // When parsing a data response, we use the permissive mode of the DMR parser
96  // (which allows Map elements to reference Arrays that are not in the DMR).
97  // Do not use that mode when parsing the DMR response - assume the DMR is
98  // valid. jhrg 4/13/16
99  parser.intern(*rs.get_cpp_stream(), &dmr);
100  }
101  catch (Error &e) {
102  cerr << "Exception: " << e.get_error_message() << endl;
103  return;
104  }
105  catch (std::exception &e) {
106  cerr << "Exception: " << e.what() << endl;
107  return;
108  }
109  catch (...) {
110  cerr << "Exception: unknown error" << endl;
111  return;
112  }
113 
114  return;
115  }
116 
117  default:
118  throw Error("Unknown response type");
119  }
120 }
121 
124 void D4Connect::process_data(DMR &data, Response &rs)
125 {
126  DBG(cerr << "Entering D4Connect::process_data" << endl);
127 
128  assert(rs.get_cpp_stream()); // DAP4 code uses cpp streams
129 
130  data.set_dap_version(rs.get_protocol());
131 
132  DBG(cerr << "Entering process_data. Response.getVersion() = " << rs.get_version() << endl);
133  switch (rs.get_type()) {
134  case dap4_error: {
135 #if 0
136  Error e;
137  if (!e.parse(rs.get_cpp_stream()))
138  throw InternalErr(__FILE__, __LINE__, "Could not parse the Error object returned by the server!");
139  throw e;
140 #endif
141  throw InternalErr(__FILE__, __LINE__, "DAP4 errors not processed yet: FIXME!");
142  }
143 
144  case web_error:
145  // Web errors (those reported in the return document's MIME header)
146  // are processed by the WWW library.
147  throw InternalErr(__FILE__, __LINE__,
148  "An error was reported by the remote httpd; this should have been processed by HTTPConnect..");
149 
150  case dap4_data: {
151 #if BYTE_ORDER_PREFIX
152  // Read the byte-order byte; used later on
153  char byte_order;
154  *rs.get_cpp_stream() >> byte_order;
155  //if (debug) cerr << "Byte order: " << ((byte_order) ? "big endian" : "little endian") << endl;
156 #endif
157  // get a chunked input stream
158 #if BYTE_ORDER_PREFIX
159  chunked_istream cis(*rs.get_cpp_stream(), 1024, byte_order);
160 #else
161  chunked_istream cis(*(rs.get_cpp_stream()), CHUNK_SIZE);
162 #endif
163  // parse the DMR, stopping when the boundary is found.
164  try {
165  // force chunk read
166  // get chunk size
167  int chunk_size = cis.read_next_chunk();
168  if (chunk_size < 0)
169  throw Error("Found an unexpected end of input (EOF) while reading a DAP4 data response. (1)");
170 
171  // get chunk
172  char chunk[chunk_size];
173  cis.read(chunk, chunk_size);
174  // parse char * with given size
175  D4ParserSax2 parser;
176  // permissive mode allows references to Maps that are not in the response.
177  // Use this mode when parsing a data response (but not the DMR). jhrg 4/13/16
178  parser.set_strict(false);
179 
180  // '-2' to discard the CRLF pair
181  parser.intern(chunk, chunk_size - 2, &data);
182  }
183  catch (Error &e) {
184  cerr << "Exception: " << e.get_error_message() << endl;
185  return;
186  }
187  catch (std::exception &e) {
188  cerr << "Exception: " << e.what() << endl;
189  return;
190  }
191  catch (...) {
192  cerr << "Exception: unknown error" << endl;
193  return;
194  }
195 
196 #if BYTE_ORDER_PREFIX
197  D4StreamUnMarshaller um(cis, byte_order);
198 #else
199  D4StreamUnMarshaller um(cis, cis.twiddle_bytes());
200 #endif
201  data.root()->deserialize(um, data);
202 
203  return;
204  }
205 
206  default:
207  throw Error("Unknown response type");
208  }
209 }
210 
219 void D4Connect::parse_mime(Response &rs)
220 {
221  rs.set_version("dods/0.0"); // initial value; for backward compatibility.
222  rs.set_protocol("2.0");
223 
224  istream &data_source = *rs.get_cpp_stream();
225  string mime = get_next_mime_header(data_source);
226  while (!mime.empty()) {
227  string header, value;
228  parse_mime_header(mime, header, value);
229 
230  // Note that this is an ordered list
231  if (header == "content-description") {
232  DBG(cout << header << ": " << value << endl);
233  rs.set_type(get_description_type(value));
234  }
235  // Use the value of xdods-server only if no other value has been read
236  else if (header == "xdods-server" && rs.get_version() == "dods/0.0") {
237  DBG(cout << header << ": " << value << endl);
238  rs.set_version(value);
239  }
240  // This trumps 'xdods-server' and 'server'
241  else if (header == "xopendap-server") {
242  DBG(cout << header << ": " << value << endl);
243  rs.set_version(value);
244  }
245  else if (header == "xdap") {
246  DBG(cout << header << ": " << value << endl);
247  rs.set_protocol(value);
248  }
249  // Only look for 'server' if no other header supplies this info.
250  else if (rs.get_version() == "dods/0.0" && header == "server") {
251  DBG(cout << header << ": " << value << endl);
252  rs.set_version(value);
253  }
254 
255  mime = get_next_mime_header(data_source);
256  }
257 }
258 
259 // public mfuncs
260 
267 D4Connect::D4Connect(const string &url, string uname, string password) :
268  d_http(0), d_local(false), d_URL(""), d_UrlQueryString(""), d_server("unknown"), d_protocol("4.0")
269 {
270  string name = prune_spaces(url);
271 
272  // Figure out if the URL starts with 'http', if so, make sure that we
273  // talk to an instance of HTTPConnect.
274  if (name.find("http") == 0) {
275  DBG(cerr << "Connect: The identifier is an http URL" << endl);
276  d_http = new HTTPConnect(RCReader::instance());
277  d_http->set_use_cpp_streams(true);
278 
279  d_URL = name;
280 
281  // Find and store any CE given with the URL.
282  string::size_type dotpos = name.find('?');
283  if (dotpos != std::string::npos) { // Found a match.
284  d_URL = name.substr(0, dotpos);
285 
286  d_UrlQueryString = name.substr(dotpos + 1);
287 
288  if (d_UrlQueryString.find(DAP4_CE_QUERY_KEY) != std::string::npos) {
289  std::stringstream msg;
290  msg << endl;
291  msg << "WARNING: A DAP4 constraint expression key was found in the query string!" << endl;
292  msg << "The submitted dataset URL: " << name << endl;
293  msg << "Contains the query string: " << d_UrlQueryString << endl;
294  msg << "This will cause issues when making DAP4 requests that specify additional constraints. " << endl;
295  cerr << msg.str() << endl;
296  // throw Error(malformed_expr, msg.str());
297  }
298 
299  }
300  }
301  else {
302  DBG(cerr << "Connect: The identifier is a local data source." << endl);
303  d_local = true; // local in this case means non-DAP
304  }
305 
306  set_credentials(uname, password);
307 }
308 
309 D4Connect::~D4Connect()
310 {
311  if (d_http) delete d_http;
312 }
313 
314 std::string D4Connect::build_dap4_ce(const string requestSuffix, const string dap4ce)
315 {
316  std::stringstream url;
317  bool needsAmpersand = false;
318 
319  url << d_URL << requestSuffix << "?";
320 
321  if (d_UrlQueryString.length() > 0) {
322  url << d_UrlQueryString;
323  needsAmpersand = true;
324  }
325 
326  if (dap4ce.length() > 0) {
327  if (needsAmpersand) url << "&";
328 
329  url << DAP4_CE_QUERY_KEY << "=" << id2www_ce(dap4ce);
330  }
331 
332  DBG(cerr << "D4Connect::build_dap4_ce() - Source URL: " << d_URL << endl);
333  DBG(cerr << "D4Connect::build_dap4_ce() - Source URL Query String: " << d_UrlQueryString << endl);
334  DBG(cerr << "D4Connect::build_dap4_ce() - dap4ce: " << dap4ce << endl);
335  DBG(cerr << "D4Connect::build_dap4_ce() - request URL: " << url.str() << endl);
336 
337  return url.str();
338 }
339 
340 void D4Connect::request_dmr(DMR &dmr, const string expr)
341 {
342  string url = build_dap4_ce(".dmr", expr);
343 
344  Response *rs = 0;
345  try {
346  rs = d_http->fetch_url(url);
347 
348  d_server = rs->get_version();
349  d_protocol = rs->get_protocol();
350 
351  switch (rs->get_type()) {
352  case unknown_type:
353  DBG(cerr << "Response type unknown, assuming it's a DMR response." << endl);
354  /* no break */
355  case dap4_dmr: {
356  D4ParserSax2 parser;
357  parser.intern(*rs->get_cpp_stream(), &dmr);
358  break;
359  }
360 
361  case dap4_error:
362  throw InternalErr(__FILE__, __LINE__, "DAP4 errors are not processed yet.");
363 
364  case web_error:
365  // We should never get here; a web error should be picked up read_url
366  // (called by fetch_url) and result in a thrown Error object.
367  throw InternalErr(__FILE__, __LINE__, "Web error found where it should never be.");
368  break;
369 
370  default:
371  throw InternalErr(__FILE__, __LINE__,
372  "Response type not handled (got " + long_to_string(rs->get_type()) + ").");
373  }
374  }
375  catch (...) {
376  delete rs;
377  throw;
378  }
379 
380  delete rs;
381 }
382 
383 void D4Connect::request_dap4_data(DMR &dmr, const string expr)
384 {
385  string url = build_dap4_ce(".dap", expr);
386 
387  Response *rs = 0;
388  try {
389  rs = d_http->fetch_url(url);
390 
391  d_server = rs->get_version();
392  d_protocol = rs->get_protocol();
393 
394  switch (rs->get_type()) {
395  case unknown_type:
396  DBG(cerr << "Response type unknown, assuming it's a DAP4 Data response." << endl);
397  /* no break */
398  case dap4_data: {
399 #if BYTE_ORDER_PREFIX
400  istream &in = *rs->get_cpp_stream();
401  // Read the byte-order byte; used later on
402  char byte_order;
403  in >> byte_order;
404 #endif
405 
406  // get a chunked input stream
407 #if BYTE_ORDER_PREFIX
408  chunked_istream cis(*(rs->get_cpp_stream()), 1024, byte_order);
409 #else
410  chunked_istream cis(*(rs->get_cpp_stream()), CHUNK_SIZE);
411 #endif
412 
413  // parse the DMR, stopping when the boundary is found.
414 
415  // force chunk read
416  // get chunk size
417  int chunk_size = cis.read_next_chunk();
418  if (chunk_size < 0)
419  throw Error("Found an unexpected end of input (EOF) while reading a DAP4 data response. (2)");
420 
421  // get chunk
422  char chunk[chunk_size];
423  cis.read(chunk, chunk_size);
424  // parse char * with given size
425  D4ParserSax2 parser;
426  // permissive mode allows references to Maps that are not in the response.
427  parser.set_strict(false);
428  // '-2' to discard the CRLF pair
429  parser.intern(chunk, chunk_size - 2, &dmr, false /*debug*/);
430 
431  // Read data and store in the DMR
432 #if BYTE_ORDER_PREFIX
433  D4StreamUnMarshaller um(cis, byte_order);
434 #else
435  D4StreamUnMarshaller um(cis, cis.twiddle_bytes());
436 #endif
437  dmr.root()->deserialize(um, dmr);
438 
439  break;
440  }
441 
442  case dap4_error:
443  throw InternalErr(__FILE__, __LINE__, "DAP4 errors are not processed yet.");
444 
445  case web_error:
446  // We should never get here; a web error should be picked up read_url
447  // (called by fetch_url) and result in a thrown Error object.
448  throw InternalErr(__FILE__, __LINE__, "Web error found where it should never be.");
449  break;
450 
451  default:
452  throw InternalErr(__FILE__, __LINE__,
453  "Response type not handled (got " + long_to_string(rs->get_type()) + ").");
454  }
455  }
456  catch (...) {
457  delete rs;
458  throw;
459  }
460 
461  delete rs;
462 }
463 
464 void D4Connect::read_dmr(DMR &dmr, Response &rs)
465 {
466  parse_mime(rs);
467  if (rs.get_type() == unknown_type) throw Error("Unknown response type.");
468 
469  read_dmr_no_mime(dmr, rs);
470 }
471 
472 void D4Connect::read_dmr_no_mime(DMR &dmr, Response &rs)
473 {
474  // Assume callers know what they are doing
475  if (rs.get_type() == unknown_type) rs.set_type(dap4_dmr);
476 
477  switch (rs.get_type()) {
478  case dap4_dmr:
479  process_dmr(dmr, rs);
480  d_server = rs.get_version();
481  d_protocol = dmr.dap_version();
482  break;
483  default:
484  throw Error("Expected a DAP4 DMR response.");
485  }
486 }
487 
488 void D4Connect::read_data(DMR &data, Response &rs)
489 {
490  parse_mime(rs);
491  if (rs.get_type() == unknown_type) throw Error("Unknown response type.");
492 
493  read_data_no_mime(data, rs);
494 }
495 
496 void D4Connect::read_data_no_mime(DMR &data, Response &rs)
497 {
498  // Assume callers know what they are doing
499  if (rs.get_type() == unknown_type) rs.set_type(dap4_data);
500 
501  switch (rs.get_type()) {
502  case dap4_data:
503  process_data(data, rs);
504  d_server = rs.get_version();
505  d_protocol = data.dap_version();
506  break;
507  default:
508  throw Error("Expected a DAP4 Data response.");
509  }
510 }
511 
517 void D4Connect::set_credentials(string u, string p)
518 {
519  if (d_http) d_http->set_credentials(u, p);
520 }
521 
526 {
527  if (d_http) d_http->set_accept_deflate(deflate);
528 }
529 
535 void D4Connect::set_xdap_protocol(int major, int minor)
536 {
537  if (d_http) d_http->set_xdap_protocol(major, minor);
538 }
539 
544 {
545  if (d_http) d_http->set_cache_enabled(cache);
546 }
547 
548 bool D4Connect::is_cache_enabled()
549 {
550  if (d_http)
551  return d_http->is_cache_enabled();
552  else
553  return false;
554 }
555 
556 } // namespace libdap
string get_next_mime_header(FILE *in)
Definition: mime_util.cc:838
void set_credentials(std::string u, std::string p)
Set the credentials for responding to challenges while dereferencing URLs.
Definition: D4Connect.cc:517
string id2www_ce(string in, const string &allowable)
Definition: escaping.cc:178
string prune_spaces(const string &name)
Definition: util.cc:459
void set_credentials(const string &u, const string &p)
D4Group * root()
Definition: DMR.cc:360
Read data from the stream made by D4StreamMarshaller.
void set_xdap_protocol(int major, int minor)
Definition: D4Connect.cc:535
STL namespace.
void set_cache_enabled(bool enabled)
Definition: HTTPConnect.h:150
void set_strict(bool s)
Set the &#39;strict&#39; mode to true or false.
Definition: D4ParserSax2.h:297
HTTPResponse * fetch_url(const string &url)
Definition: HTTPConnect.cc:617
A class for software fault reporting.
Definition: InternalErr.h:64
void parse_mime_header(const string &header, string &name, string &value)
Definition: mime_util.cc:912
void set_accept_deflate(bool deflate)
Definition: D4Connect.cc:525
ObjectType get_description_type(const string &value)
Definition: mime_util.cc:339
void set_cache_enabled(bool enabled)
Definition: D4Connect.cc:543
virtual void deserialize(D4StreamUnMarshaller &um, DMR &dmr)
Definition: D4Group.cc:544
void set_accept_deflate(bool defalte)
Definition: HTTPConnect.cc:998
void set_xdap_protocol(int major, int minor)
A class for error processing.
Definition: Error.h:90