base
This commit is contained in:
biosvos
@@ -0,0 +1,558 @@
|
||||
###
|
||||
# $Id: CSV.pm,v 1.10 2001/08/19 01:42:33 christopher Exp $
|
||||
#
|
||||
# Distributed under the GNU Lesser General Public License v2.1. See the
|
||||
# accompanying lgpl.txt file for the license text; if the file was missing
|
||||
# you may always obtain a copy from http://www.fsf.org/.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# General Public License for more details.
|
||||
#
|
||||
# Copyright (c)2001 Christopher Rath <Christopher@Rath.ca> and
|
||||
# Mark Mielke <Mark@Mielke.cc>
|
||||
###
|
||||
|
||||
###
|
||||
# POD starts.
|
||||
#
|
||||
|
||||
=head1 NAME
|
||||
|
||||
CSV.pm - a module providing manipulation routines for comma separated value (CSV) records
|
||||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
use CSV;
|
||||
|
||||
%fieldLayout = CSVinit($firstRecord);
|
||||
if (! CSVvalidate(%CSVfields, I<field list>)) {
|
||||
die "Fields missing."};
|
||||
}
|
||||
while (<INPUT>) {
|
||||
@record = CSVsplit($_);
|
||||
I<process the records>;
|
||||
}
|
||||
|
||||
=head1 LICENSE
|
||||
|
||||
Copyright (c)2001 Christopher Rath <C<Christopher@Rath.ca>> and Mark Mielke
|
||||
<C<Mark@Mielke.cc>>.
|
||||
|
||||
Distributed under the GNU Lesser General Public License v2.1. See the
|
||||
accompanying C<lgpl.txt> file for the license text; if the file was missing you
|
||||
may always obtain a copy from C<http://www.fsf.org/>.
|
||||
|
||||
=head1 WARRANTY
|
||||
|
||||
This program is distributed in the hope that it will be useful, but B<without
|
||||
any warranty>; without even the implied warranty of B<merchantability> or
|
||||
B<fitness for a particular purpose>. See the GNU General Public License for
|
||||
more details.
|
||||
|
||||
=head1 DESCRIPTION
|
||||
|
||||
This module defines some functions for reading comma separated value (CSV)
|
||||
files (e.g., Remedy-ARS generated files). The module also allows some a
|
||||
delimiter other than a comma to be used, or a selection of delimiters to be
|
||||
used (e.g., comma and semicolon).
|
||||
|
||||
The B<CSVinit(>I<$firstRecord>B<)> function is used to parse the initial record
|
||||
of a CSV file; which by definition will contain the field names for each field
|
||||
in the CSV records which follow in the file. This initial record is itself a
|
||||
CSV record.
|
||||
|
||||
The B<CSVvalidate(>I<\%output_from_CSVinit>B<,> <I<field_list>>B<)> function
|
||||
is used to verify that the incoming file contains the full list of fields
|
||||
specified in the I<field_list> provided to the function.
|
||||
|
||||
The B<CSVsplit(>I<$rawRecord>B<)> function splits a line, passed as
|
||||
I<$rawRecord> (a CSV record), and returns it split into fields; sort-of like
|
||||
B<split()>.
|
||||
|
||||
The B<$CSV::Delimiters> variable can be set to specify an alternate, or set of
|
||||
alternate, field delimiters. In general, it should be specified as a B<local>.
|
||||
|
||||
The B<CSVjoin(>I<@fields_to_join>B<)> function takes an array of fields and
|
||||
turns them into a comma separated value record. If B<$CSV::Delimiters> has
|
||||
been used to specify multiple delimiters then B<CSVjoin()> will use the first
|
||||
delimiter of the set for creation of CSV records.
|
||||
|
||||
=head1 USAGE
|
||||
|
||||
A sample application (Demo_App) of this module has been provided in order to
|
||||
demonstrate how it can be used. Some additional usage notes follow.
|
||||
|
||||
=head2 CSV::Delimiters
|
||||
|
||||
local $CSV::Delimiters = ",;";
|
||||
%fieldLayout = CSVinit($firstRecord);
|
||||
@Fields = CSVsplit($aRawRecord);
|
||||
|
||||
This will cause either of 'B<,>' or 'B<;>' to be valid delimiters, even if
|
||||
mixed. B<local()> is used to temporarily override the value, instead of
|
||||
permanently overriding the value.
|
||||
|
||||
B<CSVjoin()> will prefer the first character in the Delimiters string, or the old
|
||||
behaviour of a ',' if the Delimiters string is not defined.
|
||||
|
||||
If you peek at the code, it actually implements the split core twice. The
|
||||
first with B<$CSV::Delimiters>, and the second with 'B<,>' hard-coded. This is
|
||||
to maintain efficiency for code that does not make use of a dynamic set of
|
||||
delimiter characters.
|
||||
|
||||
=head2 CSVinit()
|
||||
|
||||
B<CSVinit()> is used to initialize the data structures for further CSV parsing.
|
||||
The function scans the input record passed to it. If it contains data, then
|
||||
that data is used to define the field titles. If no data is passed then some
|
||||
default column data is used instead. The function returns an associative array
|
||||
containing field names and the corresponding field number.
|
||||
|
||||
=head3 Parameters
|
||||
|
||||
[1] $rawRecord - Record defining field layouts.
|
||||
|
||||
=head3 Returns
|
||||
|
||||
Returns an associative array of field layout.
|
||||
|
||||
=head2 CSVvalidate()
|
||||
|
||||
B<CSVvalidate()> is used to validate a set of field names. A list of fields is
|
||||
validated against the associative array previously built by a call to
|
||||
CSVinit(). Returns true or false.
|
||||
|
||||
=head3 Parameters
|
||||
|
||||
[1] \%fields - the fields as built by CSVinit(). (reference)
|
||||
|
||||
[?] $... - a list of field names to check.
|
||||
|
||||
=head3 Returns
|
||||
|
||||
Returns 0 if any single field is not found in %fields; otherwise 1.
|
||||
|
||||
=head2 CSVjoin()
|
||||
|
||||
B<CSVjoin()> is used to join CSV data.
|
||||
|
||||
=head3 Parameters
|
||||
|
||||
Context 1:
|
||||
|
||||
[1] \@fields - An array of fields to join into a CSV.
|
||||
[2] "minimum"|"quoteall" - Defaults to "minimum", define
|
||||
whether fields must be quoted when not req'd.
|
||||
|
||||
Context 2:
|
||||
|
||||
[?] $... - An array of fields to join into
|
||||
a CSV.
|
||||
|
||||
=head3 Returns
|
||||
|
||||
Returns a string which may be "un"join'ed using CSVsplit().
|
||||
except in the case of a newline contained within a field)
|
||||
|
||||
=head2 CSVsplit()
|
||||
|
||||
B<CSVsplit> is used to split CSV data, just like Perl's own split() does.
|
||||
|
||||
=head3 Parameters
|
||||
[1] $rawRecord - the record to split.
|
||||
|
||||
=head3 Returns
|
||||
|
||||
Returns an array of values split out of $rawRecord.
|
||||
|
||||
=head1 TESTING
|
||||
|
||||
Put the following lines into a file to test the comma-based parsing of this
|
||||
module:
|
||||
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
use CSV;
|
||||
|
||||
while (<DATA>) {
|
||||
print join(":", CSVsplit($_)), "\n";
|
||||
}
|
||||
|
||||
__DATA__
|
||||
"One from FAQ will do","sec""ond"
|
||||
"But, not","3rd""","or","""fifth"
|
||||
"don't","forget","fourth"",""","and" , """,""sixth".
|
||||
"Of course, it doesn't solve",everything,8,N,1
|
||||
"Is there a new emacs perl-mode?"
|
||||
"Tom","tom@fiction.org"
|
||||
Empty fields needed,,"and "wanted
|
||||
|
||||
Some,"",boundary ",cases"", ,too
|
||||
Grok this\, Spok!
|
||||
"Didn't you notice\! in it \{the spec\}"
|
||||
"Multi-
|
||||
line",test
|
||||
|
||||
This will produce the following output:
|
||||
|
||||
One from FAQ will do:sec"ond
|
||||
But, not:3rd":or:"fifth
|
||||
don't:forget:fourth",":"and" : """:""sixth".
|
||||
Of course, it doesn't solve:everything:8:N:1
|
||||
Is there a new emacs perl-mode?
|
||||
Tom:tom@fiction.org
|
||||
Empty fields needed::"and "wanted
|
||||
|
||||
Some::boundary ":cases"": :too
|
||||
Grok this\: Spok!
|
||||
Didn't you notice\! in it \{the spec\}
|
||||
"Multi-
|
||||
line":test
|
||||
|
||||
=head1 CSV SPECIFICATION
|
||||
|
||||
This section attempts to define CSV records and files in a fairly rigorous
|
||||
fashion. The point behind this is to make this module usable without having to
|
||||
read and understand the source code.
|
||||
|
||||
=head2 CSV Records
|
||||
|
||||
The basic idea behind a CSV record is this: literal field values are
|
||||
delimited by commas. The immediate complication that arises is, of
|
||||
course, ``What should be done when a comma must appear within a
|
||||
field?'' Within the bounds of current practice, there are two
|
||||
immediate solutions to this complication:
|
||||
|
||||
=over 4
|
||||
|
||||
=item 1.
|
||||
|
||||
Use a predefined escape character to tag commas which appear within fields.
|
||||
|
||||
=item 2.
|
||||
|
||||
Allow quotation marks to enclose a field and _protect_ a comma appearing within
|
||||
a field.
|
||||
|
||||
=back
|
||||
|
||||
As with all work-arounds, these "immediate solutions" have complications of
|
||||
their own (these secondary complications are numbered the same as their primary
|
||||
counterparts):
|
||||
|
||||
=over 4
|
||||
|
||||
=item 1.
|
||||
|
||||
Escape characters must themselves be escaped in order to appear as a value
|
||||
within a field (e.g., if a literal comma is expressed as ``\,'', then a literal
|
||||
backslash must appear as ``\\'').
|
||||
|
||||
=item 2.
|
||||
|
||||
Quotation marks must somehow be protected if they are to appear as a literal
|
||||
character within a field.
|
||||
|
||||
=back
|
||||
|
||||
Given that the essence of CSV files is simplicity, I have decided to reject
|
||||
I<all> escape and escaped characters with the exception of quoation marks
|
||||
appearing within quotation marks. That is, the case of the escaped comma has
|
||||
been rejected from this specification.
|
||||
|
||||
Within the context of Perl, the L<string(3)> library and the UNIX shells, an
|
||||
additional level of complexity is added to this equation when we begin to ask,
|
||||
``What is the meaning or significance of whitespace within a CSV record?'' The
|
||||
meaning of whitespace is a key technical detail which must be accounted for in
|
||||
both the specification and its implementation; otherwise, everyone's
|
||||
implementation will produce semi-random results based upon that implementors
|
||||
opinion regarding whitespace.
|
||||
|
||||
=head2 Semi-Formal CSV Record Specification
|
||||
|
||||
This specification uses the syntax described in S<Appendix A> of the first
|
||||
edition of O'Reilly's Programming Perl book (i.e., the S<Perl 4> camel book).
|
||||
|
||||
CSV_RECORD ::= (* FIELD DELIM *) FIELD REC_SEP
|
||||
|
||||
FIELD ::= QUOTED_TEXT | TEXT
|
||||
|
||||
DELIM ::= `,'
|
||||
|
||||
REC_SEP ::= `\n'
|
||||
|
||||
TEXT ::= LIT_STR | ["] LIT_STR [^"] | [^"] LIT_STR ["]
|
||||
|
||||
LIT_STR ::= (* LITERAL_CHAR *)
|
||||
|
||||
LITERAL_CHAR ::= NOT_COMMA_NL
|
||||
|
||||
NOT_COMMA_NL ::= [^,\n]
|
||||
|
||||
QUOTED_TEXT ::= ["] (* NOT_A_QUOTE *) ["]
|
||||
|
||||
NOT_A_QUOTE ::= [^"] | ESCAPED_QUOTE
|
||||
|
||||
ESCAPED_QUOTE ::= `""'
|
||||
|
||||
=head2 Notes
|
||||
|
||||
This specification does not grant any special status to whitespace characters.
|
||||
This means that B<I<all whitespace is part of some field value.>>
|
||||
|
||||
The B<TEXT> non-terminal is attempting to express the cases where quotation
|
||||
marks exist but do not completely encapsulate the field value; in cases like
|
||||
this, the quotation marks should be treated as literal characters making up
|
||||
part of the field value.
|
||||
|
||||
One ambiguity exist in this specification that I have been unable to properly
|
||||
express. The case of a field with the value ,abc""de,. Should the double
|
||||
quotation marks be treated as an escaped quotation mark or as two quotation
|
||||
marks? I believe that occurences of "" should be treated as escaped quotation
|
||||
marks I<only> within a quoted string.
|
||||
|
||||
The B<LITERAL_CHAR> non-terminal exists partially as a place-holder. Escaped
|
||||
characters may be easily accomadated by this specification at a later date by
|
||||
OR-ing them to the right side of B<LITERAL_CHAR>.
|
||||
|
||||
Some of the non-terminals exist solely as documentation/reading aids. The
|
||||
B<NOT_A_COMMA_NL> is one example of this case; its name helps express the
|
||||
meaning of the regex (which should assist other readers of this document to
|
||||
detect errors in the specification).
|
||||
|
||||
The B<ESCAPED_QUOTE> non-terminal includes the PASCAL-like case of ``""'' and
|
||||
excludes the more traditional UNIX ``\\"''. This is not my preference; I have
|
||||
included it here because I know there exists at least one commercial tool that
|
||||
produces CSV records containing the PASCAL-like construct and not the UNIX-like
|
||||
one.
|
||||
|
||||
=head1 AUTHORS
|
||||
|
||||
Christopher Rath (C<christopher@rath.ca>) wrote the CSV specification and
|
||||
everything in the module except the essential snippet of code that actually
|
||||
does the work :).
|
||||
|
||||
Mark Mielke (C<mark@mielke.cc>) took the specification and wrote the essential
|
||||
piece of code that actually breaks the CSV records into its constituent fields.
|
||||
He also took the initial .pl version and .pm'ed it (this only makes sense,
|
||||
since this module is only usable in perl5).
|
||||
|
||||
=head1 BUGS
|
||||
|
||||
=head2 Not Thread-Safe
|
||||
|
||||
This module, and hence B<CSVinit()>/B<CSVsplit()>, is not thread-safe or
|
||||
re-entrant.
|
||||
|
||||
=head2 Fields Spanning Lines
|
||||
|
||||
This module currently fails in one of the test-cases, although the test output
|
||||
listed herein, above, has been constructed to show the actual output of this
|
||||
module, as opposed to the correct output:
|
||||
|
||||
"Multi-
|
||||
line",test
|
||||
|
||||
This is due to the fact that perl is reading one line at a time with:
|
||||
|
||||
while (<DATA>) { ... }
|
||||
|
||||
So the first line is read ("Multi-) and evaluated. The _second_ time around the
|
||||
loop the second line (line",test) is read and evaluated. There is no
|
||||
workaround available, this is simply a limitation of the module.
|
||||
|
||||
=head1 VERSION
|
||||
|
||||
The RCS identifier for this module is
|
||||
S<$Id: CSV.pm,v 1.10 2001/08/19 01:42:33 christopher Exp $>.
|
||||
|
||||
=cut
|
||||
|
||||
#
|
||||
# End of POD.
|
||||
###
|
||||
|
||||
package CSV;
|
||||
require 5.000;
|
||||
require Exporter;
|
||||
$VERSION = 2.00;
|
||||
|
||||
@ISA = qw(Exporter);
|
||||
@EXPORT = qw(CSVinit CSVvalidate CSVjoin CSVsplit);
|
||||
|
||||
use strict;
|
||||
|
||||
$CSV::Delimiters = undef;
|
||||
|
||||
###
|
||||
# Setup Variables.
|
||||
#
|
||||
# CSVDefaultColumnMapping - this contains a dummy record; just in case the
|
||||
# user calls us with nothing.
|
||||
#
|
||||
my(%CSVDefaultColumnMapping) = ("" => -1);
|
||||
|
||||
###
|
||||
# CSVinit() - initialize data structures for further CSV parsing.
|
||||
#
|
||||
# This function scans the input record passed to it. If it contains data, then
|
||||
# that data is used to define the field titles. If no data is passed then some
|
||||
# default column data is used instead. The function returns an associative
|
||||
# array containing field names and the corresponding field number.
|
||||
#
|
||||
# Parameters;
|
||||
# [1] $rawRecord - Record defining field layouts.
|
||||
#
|
||||
# Returns:
|
||||
# Associative array of field layout.
|
||||
###
|
||||
sub CSVinit
|
||||
{
|
||||
my @fields = CSVsplit(@_);
|
||||
my %columns;
|
||||
my $i = 0;
|
||||
|
||||
if ($#fields >= 0) {
|
||||
# Build associative array with field -> column_number mapping.
|
||||
map { $columns{$_} = $i++ } @fields;
|
||||
%columns;
|
||||
} else {
|
||||
%CSVDefaultColumnMapping;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
###
|
||||
# CSVvalidate() - validate the field names passed in.
|
||||
#
|
||||
# A list of fields is validated against the associative array previously built
|
||||
# by a call to CSVinit(). Returns true or false.
|
||||
#
|
||||
# Parameters:
|
||||
# [1] \%fields - the fields as built by CSVinit(). (reference)
|
||||
# [?] $... - a list of field names to check.
|
||||
#
|
||||
# Returns:
|
||||
# 0 if any single field is not found in %fields; otherwise 1.
|
||||
###
|
||||
sub CSVvalidate
|
||||
{
|
||||
my $fields = shift; # pass by reference!
|
||||
my $field;
|
||||
|
||||
foreach $field (@_) {
|
||||
exists $fields->{$field} || return 0;
|
||||
}
|
||||
|
||||
1;
|
||||
}
|
||||
|
||||
|
||||
###
|
||||
# CSVjoin() - join CSV data.
|
||||
#
|
||||
# Parameters:
|
||||
# Context 1:
|
||||
# [1] \@fields - An array of fields to join into a CSV.
|
||||
# [2] "minimum"|"quoteall" - Defaults to "minimum", defines
|
||||
# whether fields must be quoted when not req'd.
|
||||
# Context 2:
|
||||
# [?] $... - An array of fields to join into a CSV.
|
||||
#
|
||||
# Returns:
|
||||
# returns a string which may be "un"join'ed using CSVsplit().
|
||||
# (except in the case of a newline contained within a field)
|
||||
###
|
||||
sub CSVjoin
|
||||
{
|
||||
my($fields, $mode) = ref($_[0]) ? @_ : \@_;
|
||||
my($quoteall, @quotedfields, $quotedfield);
|
||||
local $_;
|
||||
|
||||
$mode = 'minimum' if !defined($mode);
|
||||
$quoteall = ($mode eq 'quoteall') ? 1 : 0;
|
||||
|
||||
foreach (@$fields) {
|
||||
if ($quoteall || /[\",\n]/) {
|
||||
($quotedfield = $_) =~ s/\"/""/g;
|
||||
push(@quotedfields, qq{"$quotedfield"});
|
||||
} else {
|
||||
push(@quotedfields, $_);
|
||||
}
|
||||
}
|
||||
|
||||
join((defined($CSV::Delimiters) ? substr($CSV::Delimiters,0,1) : ','),
|
||||
@quotedfields);
|
||||
}
|
||||
|
||||
|
||||
###
|
||||
# CSVsplit() - split CSV data, just like Perl's own split() does.
|
||||
#
|
||||
# See description of record layout at top of this module.
|
||||
#
|
||||
# Parameters:
|
||||
# [1] $rawRecord - the record to split.
|
||||
#
|
||||
# Returns:
|
||||
# An array of values split out of $rawRecord.
|
||||
###
|
||||
sub CSVsplit
|
||||
{
|
||||
local $_ = $_[0];
|
||||
my(@fields, $t);
|
||||
|
||||
# According to the specs, a new-line is a field terminator not a field
|
||||
# delimiter. This only becomes an issue at the end of a string where
|
||||
# a new-line implies end of field, not a field delimiter. By stripping
|
||||
# the last optional new-line we allow all remaining new-lines to be
|
||||
# considered field delimiters.
|
||||
chomp;
|
||||
|
||||
###
|
||||
# Only build the array if at least one field exists. (if at least one
|
||||
# character exists, by definition, at least one field exists)
|
||||
#
|
||||
if (length($_) > 0) {
|
||||
###
|
||||
# Break the record into fields.
|
||||
# Field Delimiter == /[,\n]/ (See comment above: According...)
|
||||
my $expecting;
|
||||
if (defined $CSV::Delimiters) {
|
||||
while (m`(?=.)((?:"((?:""|[^"]+)*)")?([^$CSV::Delimiters]*))([$CSV::Delimiters]?)`sg) {
|
||||
$expecting = $4;
|
||||
if (defined($2) && !length($3)) {
|
||||
($t = $2) =~ s/""/\"/g;
|
||||
push(@fields, $t);
|
||||
} else {
|
||||
push(@fields, $1);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
while (m`(?=.)((?:"((?:""|[^"]+)*)")?([^,]*))(,?)`sg) {
|
||||
$expecting = $4;
|
||||
if (defined($2) && !length($3)) {
|
||||
($t = $2) =~ s/""/\"/g;
|
||||
push(@fields, $t);
|
||||
} else {
|
||||
push(@fields, $1);
|
||||
}
|
||||
}
|
||||
}
|
||||
push(@fields, '') if $expecting;
|
||||
}
|
||||
|
||||
@fields;
|
||||
}
|
||||
|
||||
1;
|
||||
|
||||
###
|
||||
# End of package.
|
||||
###
|
||||
|
||||
# Local Variables:
|
||||
# fill-column: 79
|
||||
# End:
|
||||
Reference in New Issue
Block a user