$versions{'database-ext_lib.pl'} = '06.6.00.0001';

# Contains all sub routines (functions) for the sore manager layout and display settings manager.

#######################################################################
#
# AgoraCart and all associated files, except where noted, are
# Copyright 2001 to Present jointly by K-Factor Technologies, Inc.
# and by C E Mayo (aka Mister Ed) at AgoraCart.com & K-Factor.net
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

# http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This copyright notice may not be removed or altered in any way.
#
#######################################################################

# Library originally assembled by Steve Kneizys to allow for the CSV
# import and export of the product database via manager.cgi
# First created January 21, 2000
# Many additions, fixed updates, and import/export changes from 2001 - 2019
# by Mister Ed @ AgoraCart.com

#use table;
#use tabletypes;

{
    local ($modname) = 'database';
    register_extension( $modname, 'Database', $versions{$modname} );
    register_menu(
        'UpdateTags', "STD_action_update_tags",
        $modname,  "Update Image Tags"
    );
    register_menu(
        'ConvertProduct', 'STD_flatfile_action_convert_product',
        $modname,   'Convert v3.x thru v6.0x database to Route 66 5 user DB'
    );
    register_menu(
        'ConvertTokens', 'STD_flatfile_token_convert',
        $modname,  'Convert Database Tokens'
    );
    register_menu(
        'data_to_CSV', 'STD_action_data_to_CSV',
        $modname,  'Create CSV file of a database'
    );
    register_menu(
        'data_to_TAB', 'STD_action_data_to_TAB',
        $modname,  'Create TAB file of a database'
    );
    register_menu(
        'data_to_PIPE', 'STD_action_data_to_PIPE',
        $modname,   'Create PIPE file of a database'
    );
    register_menu(
        'data_from_CSV', 'STD_action_data_from_CSV',
        $modname,   'Import CSV database'
    );
    register_menu(
        'data_from_PIPE', 'STD_action_data_from_PIPE',
        $modname,   'Import PIPE database'
    );
    register_menu(
        'data_from_TAB', 'STD_action_data_from_TAB',
        $modname,   'Import TAB delimited database'
    );
    register_menu(
        'RemoveDBunderscores', 'remove_underscores_in_DB',
        $modname,   'Remove Underscores in DB, Fix images'
    );
    register_menu( 'database_screen', 'STD_agora_database_screen', $modname,
        'Database Screen' );
    register_menu( 'action_data_from_file', 'STD_action_data_from_file',
        $modname, 'BASE -- required to import a file' );
    register_menu( 'action_data_to_file', 'STD_action_data_to_file', $modname,
        'BASE -- required to export a file' );

    $datafile      = "$sc_data_file_path";
    $datafile_conv = $datafile;
    open( OLDFILE, "$datafile" );
    @lines = <OLDFILE>;
    @my_items = split( /\|/, $lines[0] );

    # Route 66 5 user - (21 fields)
    if ( $#my_items == 20 ) {
        add_settings_choice( 'database', 'Database Manager',
            'database_screen' );
    }
    # Route 66 10 user  - (26 fields), we have a problem
    elsif ( $#my_items == 25 ) {
        add_item_to_manager_menu( 'Warning: 10 user DB used with 5 user software, please upgrade software to Route66 10 user DB version', '', '' );
    }
    # Route 66 20 user  - (36 fields), we have a problem
    elsif ( $#my_items == 35 ) {
        add_item_to_manager_menu( 'Warning: 20 user DB used with 5 user software, please upgrade to software Route66 20 user DB version', '', '' );
    }
    # Legacy 10+4 user - (22 fields), we have a problem
    elsif ( $#my_items == 21 ) {
        add_item_to_manager_menu( 'Warning: Legacy 10+4 user DB used with Route 66 5 user software, please upgrade software to Route66 10 user DB version', '', '' );
    }
    # Legacy 20 user - (32 fields), we have a problem
    elsif ( $#my_items == 31 ) {
        add_item_to_manager_menu( 'Warning: Legacy 20 user DB used with Route 66 5 user software, please upgrade software to Route66 20 user DB version', '', '' );
    }
    # Legacy 10 user - (18 fields), we have a problem
    elsif ( $#my_items == 17 ) {
        add_item_to_manager_menu( 'Warning: Legacy 10 user DB used with Route 66 5 user software, please upgrade software to Route66 10 user DB version', '', '' );
    }
    # Legacy 5 user - (13 fields), we are okay to convert!
    elsif ( $#my_items eq 12 ) {
        add_item_to_manager_menu( 'Convert v4.x thru v6.0x 5 user DB to Route 66 5 user DB', 'ConvertProduct=yes', '' );
    }
    else {
        add_item_to_manager_menu( 'Warning, product DB format not recognizable', '', '' );
    }

    if ( ( $lines[0] =~ /%%userField/i ) || ( $lines[0] =~ /%%IMG/i ) || ( $lines[0] =~ /%%cart/i )  || ( $lines[0] =~ /%%URLof/i ) ) {
        add_item_to_manager_menu( 'Convert Database Tokens', 'ConvertTokens=yes', '' );
    }
    close(OLDFILE);

}
$mc_db_convert_per_pass = 500;    # max records to convert per pass
$mc_db_convert_seconds  = 10;     # max time to plug away per pass

# mainly for delayed initializations when file locking routines are required
codehook( 'manager_database_lib_init' );

#######################################################################
# Converts %%token%% tokens in database to [[token]]
# Converts only those needed as some tokens still use %% for options
# added by Mister Ed @ AgoraCart.com - Jan 18, 2011
#######################################################################

sub STD_flatfile_token_convert {

    $datafile_conv = $datafile;

    #create backup file.  added by Mister Ed July 14, 2006
    my $filename5 = $datafile;
    $filename5 =~ s/file$//;
    $filename5 = "$filename5" . time; # for backup DB name, incase something goes wrong.
    copy( $datafile, $filename5 );

    open( OLDFILE, "$datafile" ) || my_die("Can't Open $datafile");

    @lines = <OLDFILE>;

    $conv_result .= "Proceeding with conversion ... \n";

    open( NEWFILE, ">$datafile_conv" ) || my_die("Can't Open $datafile");

    foreach $the_whole_page (@lines) {
        $the_whole_page =~ s/%%image%%/\[\[image\]\]/ig;
        $the_whole_page =~ s/%%IMG%%/\[\[IMG\]\]/ig;
        $the_whole_page =~ s/%%cartlink%%/\[\[cartlink\]\]/ig;
        $the_whole_page =~ s/%%description%%/\[\[description\]\]/ig;
        $the_whole_page =~ s/%%Qty%%/\[\[Qty\]\]/ig;
        $the_whole_page =~ s/%%userFieldOne%%/\[\[userFieldOne\]\]/ig;
        $the_whole_page =~ s/%%userFieldTwo%%/\[\[userFieldTwo\]\]/ig;
        $the_whole_page =~ s/%%userFieldThree%%/\[\[userFieldThree\]\]/ig;
        $the_whole_page =~ s/%%userFieldFour%%/\[\[userFieldFour\]\]/ig;
        $the_whole_page =~ s/%%userFieldFive%%/\[\[userFieldFive\]\]/ig;
        $the_whole_page =~ s/%%scriptURL%%/\[\[scriptURL\]\]/ig;
        $the_whole_page =~ s/%%StepOneURL%%/\[\[StepOneURL\]\]/ig;
        $the_whole_page =~ s/%%CartID%%/\[\[CartID\]\]/ig;
        $the_whole_page =~ s/%%cart_id%%/\[\[cart_id\]\]/ig;
        $the_whole_page =~ s/%%ppinc%%/\[\[ppinc\]\]/ig;
        $the_whole_page =~ s/%%maxp%%/\[\[maxp\]\]/ig;
        $the_whole_page =~ s/%%page%%/\[\[page\]\]/ig;
        $the_whole_page =~ s/%%p_id%%/\[\[p_id\]\]/ig;
        $the_whole_page =~ s/%%keywords%%/\[\[keywords\]\]/ig;
        $the_whole_page =~ s/%%next%%/\[\[next\]\]/ig;
        $the_whole_page =~ s/%%alt_next%%/\[\[alt_next\]\]/ig;
        $the_whole_page =~ s/%%exact_match%%/\[\[exact_match\]\]/ig;
        $the_whole_page =~ s/%%exact_case%%/\[\[exact_case\]\]/ig;
        $the_whole_page =~ s/%%image%%/\[\[image\]\]/ig;
        $the_whole_page =~ s/%%price%%/\[\[price\]\]/ig;
        $the_whole_page =~ s/%%cost%%/\[\[cost\]\]/ig;
        $the_whole_page =~ s/%%shipping%%/\[\[shipping\]\]/ig;
        $the_whole_page =~ s/%%SpecialsPrice%%/\[\[SpecialsPrice\]\]/ig;
        $the_whole_page =~ s/%%WholesalePrice%%/\[\[WholesalePrice\]\]/ig;
        $the_whole_page =~ s/%%Inventory%%/\[\[Inventory\]\]/ig;
        $the_whole_page =~ s/%%CategoryID%%/\[\[CategoryID\]\]/ig;
        $the_whole_page =~ s/%%URLofImages%%/\[\[URLofImages\]\]/ig;
        $the_whole_page =~ s/%%TemplateName%%/\[\[TemplateName\]\]/ig;
        $the_whole_page =~ s/%%sc_money_symbol%%/\[\[sc_money_symbol\]\]/ig;
        $the_whole_page =~ s/%%ButtonSetURL%%/\[\[ButtonSetURL\]\]/ig;
        $the_whole_page =~ s/%%product%%/\[\[product\]\]/ig;
        $the_whole_page =~ s/%%name%%/\[\[name\]\]/ig;

        print NEWFILE $the_whole_page;

    }

    close(NEWFILE);
    $conv_result .= "Done!<br>\n";
    $conv_result .= 'If using a database other than "flat file" you will need to import the PIPE file that was just ';
    $conv_result .= "created (data.file in the $sc_data_file_dir/db/ directory.)\n";

    close(OLDFILE);
    $zitem = $menu_items{"database_screen"};
    &$zitem;

}

#######################################################################

sub STD_flatfile_action_convert_product {

    local ($URL_of_images_directory) = '[[URLofImages]]';

    $datafile_conv = $datafile;

    open( OLDFILE, "$datafile" ) || my_die("Can't Open $datafile");

    @lines = <OLDFILE>;
    @my_items = split( /\|/, $lines[0] );

    # $#my_items is last element ID, starting from 0.
    # So 12 = 13 elements, the standard v4-v6 5 user DB
    if ( $#my_items ne 12 ) {
        $conv_result =
          "This file doesn't have fields 0 to 12, it has 0 to $#my_items.\n";
        $conv_result .= "<br>Perhaps it has already been converted?\n";
    }
    else {

        $conv_result .= "Proceeding with conversion ... \n";

        open( NEWFILE, ">$datafile_conv" ) || my_die("Can't Open $datafile");

        foreach $line (@lines) {
            my $myline = '';
            chomp $line;
            @st = split( /\|/, $line );

            $image = $st[4];

            $image = parse_image_string($image);
            if ( $image eq '' ) {
                $image = 'notavailable.gif';
            }
            $formatted_image = create_image_string($image);

            if ( $st[0] > 0 ) {
                $st[0] = pad_key( ( 0 + $st[0] ), $sc_prod_db_pad_length );
            }
            $st[4] = $formatted_image;

            $myline = "$st[0]|$st[1]|$st[2]|$st[3]|$st[4]|$st[5]|$st[6]|$st[7]user1|$st[8]|$st[9]|$st[10]|$st[11]|||||||||$st[12]\n";

            print NEWFILE $myline;

        }

        close(NEWFILE);
        $conv_result .= 'Done!<br>IMG tags and PAD LENGTH were also updated.\n';
        $conv_result .= '<br>If using a database other than "flat file" you will need to import the PIPE file that was just ';
        $conv_result .= "created ( at $sc_data_file_path )\n";
    }

    close(OLDFILE);
    init_convert_menu_item();
    $zitem = $menu_items{"database_screen"};
    &$zitem;

}

#######################################################################

sub update_image_tag {
    local ($image) = @_;
    $image = parse_image_string($image);
    if ( $image eq '' ) {
        $image = 'notavailable.gif';
    }
    $image = create_image_string($image);
    return $image;
}

#######################################################################

sub STD_action_update_tags {

    local ( $zitem, $line, $raw_line, $newfile );
    local ($URL_of_images_directory) = '[[URLofImages]]';

    my $temp_start_time = time();
    my $tempname = "$sc_data_file_path";
    my $tempbackupname = "$sc_data_file_dir/db/DBproduct_backup_$temp_start_time.csv";
    copy( $tempname, $tempbackupname );


    open( DATAFILE, $sc_data_file_path ) || &file_open_error( $sc_data_file_path, " $agora_read_db_error_title ",__FILE__, __LINE__ );
    &get_file_lock("$sc_data_file_path.lock");
    while ( ( $raw_line = <DATAFILE> ) ) {
        chomp($raw_line);
        @st = split( /\|/, $raw_line );
        $image = $st[4];
        $image = parse_image_string($image);

        if ( $image eq '' ) {
            $image = 'notavailable.gif';
        }
        $formatted_image = create_image_string($image);

        if ( $st[0] > 0 ) {    # pad if it is a positive number only
            $st[0] = pad_key( ( 0 + $st[0] ), $sc_prod_db_pad_length );
        }
        $st[4] = $formatted_image;

        $st[$#db] .= q{};       # optional, force the min length to be all fields
        $myline = join( '|', @st );
        $newfile .= $myline . "\n";
    }
    close(DATAFILE);
    &release_file_lock("$sc_data_file_path.lock");
    &get_file_lock("$sc_data_file_path.lock");
    open( NEW, "> $sc_data_file_path" ) || &file_open_error( $sc_data_file_path, " $agora_read_db_error_title ",__FILE__, __LINE__ );
    print( NEW "$newfile" );
    close(NEW);
    &release_file_lock("$sc_data_file_path.lock");

    $conv_result .= "Database Image Tags and Pad Length Updated!!\n";
    init_convert_menu_item();
    $zitem = $menu_items{'database_screen'};
    &$zitem;

}

#######################################################################

sub remove_underscores_in_DB {

    local ( $zitem, $line, $raw_line, $newfile, $product, $subcat, $image );

    my $temp_start_time = time();
    my $tempname = "$sc_data_file_path";
    my $tempbackupname = "$sc_data_file_dir/db/DBproduct_backup_$temp_start_time.csv";
    copy( $tempname, $tempbackupname );


    open( DATAFILE, $sc_data_file_path ) || &file_open_error( $sc_data_file_path, " $agora_read_db_error_title ",__FILE__, __LINE__ );
    &get_file_lock("$sc_data_file_path.lock");
    while ( ( $raw_line = <DATAFILE> ) ) {
        chomp($raw_line);
        @st = split( /\|/, $raw_line );

        # fix image
        $image = $st[4];
        $image = parse_image_string($image);
        if ( $image eq '' ) {
            $image = 'notavailable.gif';
        }
        $formatted_image = create_image_string($image);
        $st[4] = $formatted_image;

        # fix category
        $product = $st[2];
        $product =~ s/_/ /g;
        $st[2] = $product;

        #fix subcats if enabled
        if ( $in{'mc_remove_subcat_underscores'} eq 'yes' ) {
            $subcat = $st[ $db{$sc_subcat_index_field} ];
            $subcat =~ s/_/ /g;
            $st[ $db{$sc_subcat_index_field} ] = $subcat;
        }

        $st[$#db] .= q{};   # optional, force the min length to be all fields
        $myline = join( '|', @st );
        $newfile .= $myline . "\n";
    }
    close(DATAFILE);
    &release_file_lock("$sc_data_file_path.lock");
    &get_file_lock("$sc_data_file_path.lock");
    open( NEW, "> $sc_data_file_path" ) || &file_open_error( $sc_data_file_path, " $agora_read_db_error_title ",__FILE__, __LINE__ );
    print( NEW "$newfile" );
    close(NEW);
    &release_file_lock("$sc_data_file_path.lock");

    $conv_result .= "Underscores Removed &amp; Images Fixed/Updated!!\n";
    init_convert_menu_item();
    $zitem = $menu_items{'database_screen'};
    &$zitem;

}

#######################################################################

sub STD_action_data_to_CSV {
    my $thing_to_do = $menu_items{'action_data_to_file'};
    &$thing_to_do('CSV');
}

#######################################################################

sub STD_action_data_to_TAB {
    my $thing_to_do = $menu_items{'action_data_to_file'};
    &$thing_to_do('TAB');
}

#######################################################################

sub STD_action_data_to_PIPE {
    my $thing_to_do = $menu_items{'action_data_to_file'};
    &$thing_to_do('PIPE');
}

#######################################################################

sub STD_action_data_to_file {
    local ($conv_type) = @_;
    local ( %db_ele, $sku, $category, %category_list, @db_row, $datafile_conv,
        $row );
    local ( $zitem, $line, $raw_line, $db_name );

    # Outputs the data.file in CSV format

    $in{'db_name'} =~ /([\w-.\/]+)/;
    $db_name = $1;
    if ( $db_name eq '' ) {
        $db_name = 'PRODUCT';
    }
    if ( $db_name ne 'PRODUCT' ) {
        my $tempname = "$sc_data_file_dir/db/" . $db_name . ".file";
        $db_file_defs{$db_name} = "$tempname";
    }

    if ( $in{'CSV_filename'} ) {
        $datafile_conv = "./$sc_data_file_dir/db/$in{'CSV_filename'}";
        $datafile_conv =~ /([\w-.\/]+)/;
        $datafile_conv = $1;
    }
    else {
        $datafile_conv = "./$sc_data_file_dir/db/database.csv";
    }

    open( NEWFILE, ">$datafile_conv" ); # || &my_CSV_file_error($datafile_conv);

    # start SQL if - else
    if ( $sc_database_lib eq 'sql_agora_db_lib.pl' ) {
        $database = connectdb( $sqldb, $sqlusername, $sqlpasswd );
        $query = runsql( "SELECT * FROM " . $sqltable . " WHERE 1", $database );
        while (
            (
                $sku,               $category,        $price,
                $short_description, $image,           $long_description,
                $shipping_price,    $userDefinedOne,  $userDefinedTwo,
                $userDefinedThree,  $userDefinedFour, $userDefinedFive,
                $SpecialsPrice, $COGSitem, $WholesalePrice, $Inventory,
                $IsTaxable, $TaxTypeID, $SpecialTaxRateAdjust, $ShipRateAdjust,
                $options
            )
            = $query->fetchrow()
          )
        {
            $string[$i] =
                $sku . '|'
              . $category . '|'
              . $price . '|'
              . $short_description . '|'
              . $image . '|'
              . $long_description . '|'
              . $shipping_price . '|'
              . $userDefinedOne . '|'
              . $userDefinedTwo . '|'
              . $userDefinedThree . '|'
              . $userDefinedFour . '|'
              . $userDefinedFive . '|'
              . $SpecialsPrice . '|'
              . $COGSitem . '|'
              . $WholesalePrice . '|'
              . $Inventory . '|'
              . $IsTaxable . '|'
              . $TaxTypeID . '|'
              . $SpecialTaxRateAdjust . '|'
              . $ShipRateAdjust . '|'
              . $options;
            $i++;
        }

        $size = @string;

        for ( $j = 0 ; $j < $size ; $j++ ) {
            $raw_line = $string[$j];
            chomp($raw_line);
            if ( $conv_type =~ /CSV/i ) {
                $line = conv_to_CSV($raw_line);
            }
            else {
                $line = $raw_line;
                if ( $conv_type =~ /TAB/i ) {
                    $line =~ s/\|/\t/g;
                }
            }
            print NEWFILE "$line\n";
        }
    }
    else {

        # SQL not used, now use flatfile in normal mode

        foreach $row ( sort ( get_db_keys($db_name) ) ) {
            get_db_row( $db_name, $row, *db_row, *raw_line, 'no' );
            chomp($raw_line);
            if ( $conv_type =~ /CSV/i ) {
                $line = conv_to_CSV($raw_line);
            }
            else {
                $line = $raw_line;
                if ( $conv_type =~ /TAB/i ) {
                    $line =~ s/\|/\t/g;
                }
            }
            print NEWFILE "$line\n";
        }
    }
    close(NEWFILE);
    $conv_result .= "The $db_name database converted to $conv_type format "
      . "in file: $datafile_conv\n";

    close(OLDFILE);
    init_convert_menu_item();

    $zitem = $menu_items{'database_screen'};
    &$zitem;

}

#######################################################################

sub my_CSV_file_error {
    local ($file) = @_;
    local ($zitem);
    $conv_result .= "ERROR: file $file could not be opened<br>\n";
    init_convert_menu_item();
    $zitem = $menu_items{'database_screen'};
    &$zitem;
    exit;
}

#######################################################################

sub STD_action_data_from_file {
    local ($conv_type) = @_;
    local ( $save_conv_type, $inx, $first, $last, $last_converted );
    local ( $last_pos, $qty, $start_tm );
    local ( $pid, $other_stuff, $line, @lines, $zitem, $orig_line );
    local ( $elapsed_mins, $orig_start_time, @st, $myline, $myaction );

    # Inputs the database in CSV, PIPE, or TAB format

    $in{'db_name'} =~ /([\w-.\/]+)/;
    $db_name = $1;
    if ( $db_name eq '' ) {
        $db_name = 'PRODUCT';
    }
    $orig_start_time = $in{'mytime'};
    $last_pos        = $in{'db_pos'};
    $start_tm        = time();
    $first           = 1 + $in{'db_record'};
    $last            = $first + $mc_db_convert_per_pass - 1;
    $inx             = 0;
    $last_converted  = $in{'db_record'};
    if ( $in{'CSV_filename'} ) {
        $datafile_conv = "$sc_data_file_dir/db/$in{'CSV_filename'}";
        open( OFILE, "$datafile_conv" ) || my_CSV_file_error($datafile_conv);
        my $tempname = "$sc_data_file_path";
        my $tempbackupname = "$sc_data_file_dir/db/DBproduct_backup_$start_tm.csv";
        if ( $db_name ne 'PRODUCT' ) {
            $tempname = "$sc_data_file_dir/db/" . $db_name . ".file";
            $tempbackupname = "$sc_data_file_dir/db/" . $db_name . "_backup_$start_tm.csv";
            $db_file_defs{$db_name} = "$tempname";
        }
        if ( $first == 1 ) {
            $orig_start_time = time();
            if ( $in{'INIT_DB'} ) {
                copy( $tempname, $tempbackupname );
                init_a_database( $db_name, '' );
            }
        }
        if ( $last_pos > 0 ) {
            $inx = $in{'db_record'};
            seek( OFILE, $last_pos, 0 );
        }
        while (<OFILE>) {
            $line      = $_;
            $orig_line = $line;
            $inx++;
            if (   ( $inx >= $first )
                && ( $inx <= $last )
                && ( ( time() - $start_tm ) < $mc_db_convert_seconds ) )
            {
                $last_pos = tell(OFILE);
                chop($line);
                $last_converted = $inx;
                codehook( 'mgr_custom_conversion_logic_1' );
                if ( $conv_type eq 'TAB' ) {
                    $line =~ s/\t/\|/g;
                }
                if ( $conv_type eq 'CSV' ) {
                    $line = conv_from_CSV($line);
                }
                codehook( 'mgr_custom_conversion_logic_2');
                @st = split( /\|/, $line );
                $pid = $st[0];
                codehook( 'mgr_custom_conversion_logic_3' );
                if ( $db_name ne 'PRODUCT' ) {
                    codehook( 'mgr_custom_conversion_logic_not_product' );
                    $myline = $line;
                }
                else {
                    $st[0] =~ s/\ /\_/g;
                    #$st[1] =~ s/\ /\_/g;

                    #      $st[1] =~ s/\-/\_/g;
                    if (   ( $in{'CSV_update_padding'} =~ /y/i )
                        && ( $st[0] > 0 ) )
                    {    # pad
                        $st[0] =
                          pad_key( ( 0 + $st[0] ), $sc_prod_db_pad_length );
                    }
                    if ( $in{'CSV_update_image_tag'} ) {
                        $st[4] = update_image_tag( $st[4] );
                    }
                    codehook( 'mgr_custom_conversion_logic_4' );
                    $st[$#db] .= q{};

                    # begin SQL
                    if ( $sc_database_lib eq 'sql_agora_db_lib.pl' ) {
                        $myline = join( ',', @st );
                    }
                    else {
                        $myline = join( '|', @st );
                    }

                    # end SQL
                    codehook( 'mgr_custom_conversion_logic_5' );
                }
                if ( $sc_database_lib eq 'sql_agora_db_lib.pl' ) {
                    addrow(
                        $st[0],  $st[1],  $st[2], $st[3], $st[4],
                        $st[5],  $st[6],  $st[7], $st[8], $st[9],
                        $st[10], $st[11], $st[12], $st[13],$st[14],
                        $st[15], $st[16], $st[17], $st[18], $st[19], $st[20]
                    );
                }
                else { put_db_raw_line( $db_name, $st[0], $myline, 'no' ); }
            }
            else {
                if ( $in{'db_pos'} > 0 ) {
                    seek( OFILE, 0, 2 );
                }
            }
        }
        $elapsed_mins = int( ( 5 + time() - $orig_start_time ) / 6 ) / 10;
        $conv_result .= "$db_name database converted ";
        $conv_result .= "from $conv_type format in file: $datafile_conv\n";
        $conv_result .= "<br>Converted $last_converted records, elapsed time: $elapsed_mins mins.\n";
    }
    else {
        $conv_result .= "Database name not specified for import!\n";
    }
    close(OFILE);
    if ( $first >= $inx ) {
        init_convert_menu_item();
        $zitem = $menu_items{'database_screen'};
        &$zitem;
    }
    else {

        # print out a reload page ...
        $myaction = "CSV_filename=$in{'CSV_filename'}";
        $myaction .= "&CSV_update_image_tag=$in{'CSV_update_image_tag'}";
        $myaction .= "&CSV_update_padding=$in{'CSV_update_padding'}";
        $myaction .= "&db_record=$last_converted";
        $myaction .= "&db_pos=$last_pos";
        $myaction .= "&db_name=$db_name";
        $myaction .= "&mytime=" . $orig_start_time;
        codehook( 'mgr_custom_conversion_myaction' );

        if ( $conv_type eq 'CSV' ) {
            $myaction .= '&data_from_CSV=YES';
        }
        if ( $conv_type eq 'PIPE' ) {
            $myaction .= '&data_from_PIPE=YES';
        }
        if ( $conv_type eq 'TAB' ) {
            $myaction .= '&data_from_TAB=YES';
        }
        $qty = $last_converted;
        if ( $qty > $inx ) { $qty = $inx; }
        $elapsed_mins = int( ( 5 + time() - $orig_start_time ) / 6 ) / 10;
        print <<ENDOFTEXT;
<html>
<head>
<title>$conv_type $db_name Database Conversion in progress . . .</title>
<script type="text/javascript">
  function myreload()
  {
  location.href="manager.cgi?mgr_seshun=$in{'mgr_seshun'}&$myaction";
  }
</script>
</head>

<body bgcolor='#F0FFFF' onLoad="myreload()">
<br><br>
<b><h2>Imported $qty records for $db_name database so far ...</h2></b><br>
<br>
Elapsed time in minutes: $elapsed_mins<br><br>
<pre>
The web browser should automatically reload for the next page ... if your browser does not reconnect for the next group of records automatically, you can <a href="manager.cgi?mgr_seshun=$in{'mgr_seshun'}&$myaction">Click Here</a>.
</pre>
<br>
</body>
</html>
ENDOFTEXT
    }
    category_list_builder_for_store();
}

#######################################################################

sub STD_action_data_from_TAB {
    my $thing_to_do = $menu_items{'action_data_from_file'};
    &$thing_to_do('TAB');
}

#######################################################################

sub STD_action_data_from_PIPE {
    my $thing_to_do = $menu_items{'action_data_from_file'};
    &$thing_to_do('PIPE');
}

#######################################################################

sub STD_action_data_from_CSV {
    my $thing_to_do = $menu_items{'action_data_from_file'};
    &$thing_to_do('CSV');
}

#######################################################################

sub STD_agora_database_screen {
    local ( $my_db_name_list, %myhash, $inx, $myopt, $count );

    print &$manager_page_header( 'Database', '', '', '', '' );

    %myhash            = %db_file_defs;
    $my_db_name_list   = q{};
    $myhash{'PRODUCT'} = 1;
    $myhash{'shipping_dimensions'} = 1;
    $count             = 0;
    foreach $inx ( sort( keys %myhash ) ) {
        $count++;
        $myopt = q{};
        if ( $inx =~ /product/i ) { $myopt .= ' selected'; }
        $my_db_name_list .= "<option$myopt>" . $inx . "</option>\n";
    }

    $my_db_name_list = qq|
        "Type of database to import/export:<span style="padding-left: 20px;"></span>\n|
      . "<select name='db_name'>\n"
      . $my_db_name_list
      . "</select>\n";

    $my_db_name_list =
      "<p>\n" . $my_db_name_list . "</p>\n";

    print <<ENDOFTEXT;
<h1 class='mgr-page-title'>Database Manager</h1>
<p><br></p>

<p class="well"><b>WARNING: THESE FUNCTIONS CAN AND DO ALTER THE ENTIRE DATABASE! MAKE SURE YOU HAVE A GOOD BACKUP OF YOUR DATA FIRST!!</b></p>
<p><br></p>
$mc_database_extra_options
ENDOFTEXT

    if ( $in{'system_edit_success'} ) {
        category_list_builder_for_store();
        print <<ENDOFTEXT;
<div class="row col-xs-12 col-sm-12 col-md-12 col-lg-12 hidden-xs">
    <div class="col-xs-12 col-sm-12 col-md-12 col-lg-12 bg-success">
        <h4 class="text-primary mgr-center">$db_update_message</h4>
        </div>
    </div>
<div class="row col-xs-12 col-sm-12"><p><br></p></div>
ENDOFTEXT
    }

    if ( $conv_result ) {
        print <<ENDOFTEXT;
<div class="row col-xs-12 col-sm-12 col-md-12 col-lg-12 hidden-xs">
    <div class="col-xs-12 col-sm-12 col-md-12 col-lg-12 bg-success">
        <h4 class="text-primary mgr-center">$conv_result</h4>
        </div>
    </div>
<div class="row col-xs-12 col-sm-12"><p><br></p></div>
ENDOFTEXT
    }

    print <<ENDOFTEXT;
<ul class="nav nav-tabs" id="tabs2" role="tablist">
    <li role="presentation" class="active"><a href="#tabs-1" aria-controls="tabs-1" role="tab" data-toggle="tab">IMPORT Database</a></li>
    <li role="presentation"><a href="#tabs-2" aria-controls="tabs-2" role="tab" data-toggle="tab">EXPORT Database</a></li>
    <li role="presentation"><a href="#tabs-3" aria-controls="tabs-3" role="tab" data-toggle="tab">Update IMG &amp; Padding in Database</a></li>
    <li role="presentation"><a href="#tabs-4" aria-controls="tabs-4" role="tab" data-toggle="tab">Remove Underscores</a></li>
</ul>

<form class="form-horizontal" action="manager.cgi" method="post">
<input type="hidden" name="mgr_seshun" value="$in{'mgr_seshun'}">
<div class="tab-content">
<div class="tab-pane active" id="tabs-1">

<p><br></p>
<h2 class="mgr-page-title"><b>IMPORT Database</b></h2>
<p><br></p>

<p class="well">You may import the PRODUCT or SHIPPING_DIMENSIONS databases (or any other defined database) in CSV, PIPE, or TAB format. TAB or PIPE delimited files are the preferred format for importing data.<br><br><b>Note:</b> an exact copy of your current data.file will be copied as a backup prior to the import process. The backup will be named in this format: DBproduct_backup_123456789.csv, where the digits stand for the server time ( highest number = newest backup ).</p>
<p><br></p>

<p><b>Name of file to Import ( REQUIRED for imports ): &nbsp; <a data-toggle="collapse" href="#importFileName" aria-expanded="false" aria-controls="importFileName"><i class="fa fa-info-circle"></i></a></b><input name="CSV_filename" class="form-control" type="text" maxlength="35"></p>
<div class="collapse" id="importFileName">
     <div class="well">
         <p>Enter the name of the CSV ( comma separated fields ), PIPE ( | separated fields ) or TAB ( tab separated fields ) formatted file to import.</p>
         <p>Files are read from the "$sc_data_file_dir/db/" sub-directory of the store.</p>
         <p>$mc_database_custom_import_instructions</p>
     </div>
</div>
<p><br></p>

$my_db_name_list
<p><hr></p>

<p>For IMPORTS you may specify if the import adds records to the current database or replaces the entire database (after backup).<br>
&nbsp;<input type="checkbox" name="INIT_DB"> Full-Import of Database (erases db before importing records)</p>

<p>&nbsp;<input type="checkbox" name="CSV_update_padding"> Update key padding with leading zeros (product db, others if so defined)</p>

<p>&nbsp;<input type="checkbox" name="CSV_update_image_tag"> Update image tag field (product db only)</p>
<p><br></p>

$mc_database_custom_import_options

<button name="data_from_CSV" class="btn btn-primary btn-lg btn-block mgr-center" value="Import records From CSV">Import records From CSV file</button>

<button name="data_from_PIPE" class="btn btn-primary btn-lg btn-block mgr-center" value="Import records From CSV">Import records From PIPE-delimited file</button>

<button name="data_from_TAB" class="btn btn-primary btn-lg btn-block mgr-center" value="Import records From CSV">Import records From TAB-delimited file</button>
<p><br></p>

$mc_database_custom_import_actions

$mc_database_extra_options_html

</form>

<p><br></p>
</div>

<!-- End of tab 1 -->

<div class="tab-pane" id="tabs-2">
<p><br></p>
<h2 class="mgr-page-title"><b>EXPORT Database</b></h2>
<p><br></p>

<p class="well">You may export the PRODUCT or SHIPPING_DIMENSIONS databases (or any other defined database) in CSV, PIPE. or TAB format.
CSV is Comma Separated Variables format. Both TAB and CSV can be used by spreadsheet and database programs as a way of importing/exporting data.</p>
<p><br></p>


<form action="manager.cgi" method="post"><input type="hidden" name="mgr_seshun" value="$in{'mgr_seshun'}">

<p><b>Name of file to Export to ( leave blank for default ): &nbsp; <a data-toggle="collapse" href="#exportFileName" aria-expanded="false" aria-controls="exportFileName"><i class="fa fa-info-circle"></i></a></b><input name="CSV_filename" class="form-control" type="text" maxlength="35"></p>
<div class="collapse" id="exportFileName">
     <div class="well">
         <p>Enter the name of the CSV ( comma separated fields ), PIPE ( | separated fields ) or TAB ( tab separated fields ) formatted file to import.</p>
         <p>If you submit a blank name, database.csv is assumed to be the name for exports.</p>
         <p><b>Note</b>: Make sure you have permission to create the file in question!</p>
         <p>Files are read from the "$sc_data_file_dir/db/" sub-directory of the store.</p>
         <p>$mc_database_custom_import_instructions</p>
     </div>
</div>
<p><br></p>

$my_db_name_list
<p><hr></p>

$mc_database_custom_export_options

<button name="data_to_CSV" class="btn btn-primary btn-lg btn-block mgr-center" value="Export entire database to CSV">Export entire database to a CSV file</button>

<button name="data_to_PIPE" class="btn btn-primary btn-lg btn-block mgr-center" value="Export entire database to PIPE">Export entire database to a PIPE file</button>

<button name="data_to_TAB" class="btn btn-primary btn-lg btn-block mgr-center" value="Export entire database to TAB">Export entire database to a TAB file</button>
<p><br></p>

$mc_database_custom_export_actions

$mc_database_extra_options_html

</form>



<p><br></p>
</div>

<!-- End of tab 2 -->

<div class="tab-pane" id="tabs-3">
<p><br></p>
<h2 class="mgr-page-title"><b>Update IMG &amp; Padding in Database</b></h2>
<p><br></p>

<p class="well">This feature updates the IMG tags the same way they would be if edited individually.  However, this button will update
all of the tags to point to the URL listed on the "Main/Core Settings" screen. It is useful if you have recently imported the database
and the pictures are not showing up, or the directory structure has been changed and you desired to place the [[URLofImages]] token in
the path. This option also sets the pad length of the product id #'s to whatever is set in the agora.setup.db file, and sorts the
records if the databse is a "flat file" so they appear in ascending order of product id.<br><br>
<b>Note:</b> an exact copy of your current data.file will be copied as a backup prior to the import process. The backup will be named in this format: DBproduct_backup_123456789.csv, where the digits stand for the server time ( highest number = newest backup ).</p>
<p><br></p>

<form action="manager.cgi" method="post">
<input type="hidden" name="mgr_seshun" value="$in{'mgr_seshun'}">
<button name="UpdateTags" class="btn btn-primary btn-lg btn-block mgr-center" value="Update IMG Tags and PAD LENGTH">Update IMG Tags and PAD LENGTH</button>
</form>

<p><br></p>
</div>

<!-- End of tab 3 -->

<div class="tab-pane" id="tabs-4">
<p><br></p>
<h2 class="mgr-page-title"><b>Remove Underscores in DB</b></h2>
<p><br></p>

<p class="well">This removes underscores in the product category field of the product DB (database) and optionally in the sub-category (subcat) field. Underscores are replaced with blank spaces.<br><br>
    <b>Note:</b> an exact copy of your current data.file will be copied as a backup prior to the import process. The backup will be named in this format: DBproduct_backup_123456789.csv, where the digits stand for the server time ( highest number = newest backup ).
</p>
<p><br></p>

<form action="manager.cgi" method="post">
<input type="hidden" name="mgr_seshun" value="$in{'mgr_seshun'}">

<p><b>Remove Underscores in Sub-categories (subcats)? &nbsp; <a data-toggle="collapse" href="#removeUnderInSubcats" aria-expanded="false" aria-controls="removeUnderInSubcats"><i class="fa fa-info-circle"></i></a></b><span style="padding-left: 25px;"></span><select name="mc_remove_subcat_underscores">
<option value="no">No</option>
<option value="yes">Yes</option>
</select>
</p>
<div class="collapse" id="removeUnderInSubcats">
  <div class="well">
    <p>Set to YES to remove underscores in the sub-categies field set in the "Main/Core Settings".</p>
    <p>Default: NO.</p>
  </div>
</div>

<p><br></p>

<button name="RemoveDBunderscores" class="btn btn-primary btn-lg btn-block mgr-center" value="Remove Underscores in DB">Remove Underscores in Product Database</button>
</form>

<p><br></p>
</div>

<!-- End of tab 4 -->

</div>
<p><br><br></p>
</div>

ENDOFTEXT
    print &$manager_page_footer;
}

#######################################################################

sub conv_to_CSV {
    local ( $in, $part ) = @_;
    local ($part);
    local ($ans) = q{};

    while ( $in ) {
        ( $part, $in ) = split( /\|/, $in, 2 );
        $part =~ s/\"/\"\"/g;
        if ( $part eq '' ) {
            $ans .= ',';
        }
        else {
            $ans .= "\"$part\",";
        }
    }

    chop($ans);
    return $ans;

}

#######################################################################
#
# sub conv_from_CSV
#  Usage: $line = &conv_from_CSV($CSV_data_line);
#
# Modified to do strings instead of arrays by SPK
#
# Based on subroutine: SplitData found at www.extropia.com
# Original info on the routine is here:
#
#   Usage:
#     @output = &SplitData($data_line);
#
#   Input:
#     $data_line = the CSV file line to be separated.
#
#   Function:
#     This subroutine is used to split a (.CSV) comma separated
#     values database line ($data_line). This subroutine was
#     written to be used with a .CSV file exported from MS Excel
#     or another Spreadsheet Program.
#
#   Output:
#     Returns line data fields in a list array format (@output).
#
#
#   Written By: Jeff Walters at hiker_jjw@yahoo.com
#   Last Edit: 7/16/99
#   Please report any problems to hiker_jjw@yahoo.com
#
#######################################################################

sub conv_from_CSV {

  #sub SplitData

    local ($data_line)   = @_;
    local ($output_line) = q{};
    local ( $combine, $combine_trigger, $data, $temp_data, $data_length,
        @split_data, @output );

    $combine_trigger = 'off';

#   Remove the new line or carriage return characters.

    $data_line =~ s/\n+$//;
    $data_line =~ s/\r+$//;

#  First, we split the data by commas into the @split_data array.

    @split_data = split( /\,/, $data_line );

#  For each data element we will check to see if the elements need to be combined.

    foreach $data (@split_data) {

        if ( $data =~ /\"/ || $combine_trigger eq 'on' ) {

            $temp_data = $data;
            $temp_data =~ s/\"{2}//g;

# Check to see if the data begins with an even or odd number of quotes
            if   ( $temp_data =~ /^\"/ ) { $beginning_quotes = 'odd' }
            else                         { $beginning_quotes = 'even' }

# Check to see if the data ends with an even or odd number of quotes
            if   ( $temp_data =~ /\"$/ ) { $ending_quotes = 'odd' }
            else                         { $ending_quotes = 'even' }

            $temp_data = $data;
            $temp_data =~ s/\"//g;
            if   ( $temp_data eq "" ) { $all_quotes = 'true' }
            else                      { $all_quotes = 'false' }

            $data_length = length($data);

            # print "Combine  \n";
            # print "$beginning_quotes  $ending_quotes  $all_quotes<br>\n";

            # $data contains a comma, but we do not need to combine.
            if (   $beginning_quotes eq 'odd'
                && $ending_quotes eq 'odd'
                && $all_quotes eq "false" )
            {
                $combine_trigger = 'off';
                $data = substr( $data, 1, $length - 1 );
                $data =~ s/\"{2}/\"/g;
                $output_line .= "$data|";
            }

            # $data is the beginning of a split field
            elsif ($beginning_quotes eq 'odd'
                && $ending_quotes eq 'even'
                && $all_quotes eq 'false' )
            {
                $combine_trigger = 'on';
                $data = substr( $data, 1 );
                $data =~ s/\"{2}/\"/g;
                $combine_data = $data;
            }

            # $data is a middle section of the split field
            elsif ($beginning_quotes eq 'even'
                && $ending_quotes eq 'even'
                && $all_quotes eq 'false' )
            {
                $combine_trigger = 'on';
                $data =~ s/\"{2}/\"/g;
                $combine_data .= ",$data";
            }

            # $data is the ending of the split field
            elsif ($beginning_quotes eq 'even'
                && $ending_quotes eq 'odd'
                && $all_quotes eq 'false' )
            {
                $combine_trigger = 'off';
                $data = substr( $data, 0, $length - 1 );
                $data =~ s/\"{2}/\"/g;
                $combine_data .= ",$data";
                $output_line  .= "$combine_data|";
                $combine_data = q{};
            }
            elsif ($beginning_quotes eq 'odd'
                && $ending_quotes eq 'odd'
                && $all_quotes eq 'true' )
            {
                $data =~ s/\"//;
                $data =~ s/\"{2}/\"/g;
                if ( $combine_trigger eq 'off' ) {
                    $combine_trigger = 'on';
                    $combine_data    = "$data";
                }
                else {
                    $combine_trigger = 'off';
                    $combine_data .= ",$data";
                    $output_line  .= "$combine_data|";
                }
            }
            elsif ($beginning_quotes eq 'even'
                && $ending_quotes eq 'even'
                && $all_quotes eq 'true' )
            {
                $data =~ s/\"{2}/\"/g;
                $combine_data .= ",$data";
            }
            else {
                $combine_trigger = 'off';
                $combine_data .= 'ERROR';
            }

        }
        else {

            # print "Regular Push<br>\n";
            $output_line .= "$data|";
        }

    }

    chop($output_line);
    return $output_line;

}

#######################################################################
#                            get_file_lock
#######################################################################

# get_file_lock is a subroutine used to create a lockfile.
# Lockfiles are used to make sure that no more than one
# instance of the script can modify a file at one time.  A
# lock file is vital to the integrity of your data.
# Imagine what would happen if two or three people
# were using the same script to modify a shared file (like
# the error log) and each accessed the file at the same
# time.  At best, the data entered by some of the users
# would be lost.  Worse, the conflicting demands could
# possibly result in the corruption of the file.
#
# Thus, it is crucial to provide a way to monitor and
# control access to the file.  This is the goal of the
# lock file routines.  When an instance of this script
# tries to  access a shared file, it must first check for
# the existence of a lock file by using the file lock
# checks in get_file_lock.
#
# If get_file_lock determines that there is an existing
# lock file, it instructs the instance that called it to
# wait until the lock file disappears.  The script then
# waits and checks back after some time interval.  If the
# lock file still remains, it continues to wait until some
# point at which the admin has given it permissios to just
# overwrite the file because some other error must have
# occurred.
#
# If, on the other hand, the lock file has dissappeared,
# the script asks get_file_lock to create a new lock file
# and then goes ahead and edits the file.
#
# The subroutine takes one argumnet, the name to use for
# the lock file and is called with the following syntax:
#
# &get_file_lock("file.name");

sub get_file_lock {

    local ($lock_file) = @_;
    local ($endtime);
    local ($exit_get_file_lock) = q{};
    codehook( 'get_file_lock' );
    if ( $exit_get_file_lock ) { return; }

    $endtime = 55;    # was 20 originally
    $endtime = time + $endtime;

    # We set endtime to wait 55 seconds.  If the lockfile has
    # not been removed by then, there must be some other
    # problem with the file system.  Perhaps an instance of
    # the script crashed and never could delete the lock file.

    while ( -e $lock_file && time < $endtime ) {
        sleep(1);
    }

    open( LOCK_FILE, ">$lock_file" )
      || CgiDie( 'I could not open the lockfile - check your permission '
          . "settings ($lock_file)" );

    # Note: If flock is available on your system, feel free to
    # use it.  flock is an even safer method of locking your
    # file because it locks it at the system level.  The above
    # routine is "pretty good" and it will serve most
    # systems.  But if you are lucky enough to have a server
    # with flock routines built in, go ahead and uncomment
    # the next line and comment the one above.

    # flock(LOCK_FILE, 2); # 2 exclusively locks the file

}

#######################################################################

sub release_file_lock {
    local ($lock_file)              = @_;
    local ($exit_release_file_lock) = q{};
    codehook( 'release_file_lock' );
    if ( $exit_release_file_lock ) { return; }

    # flock(LOCK_FILE, 8); # 8 unlocks the file
    close(LOCK_FILE);
    unlink($lock_file);
}

#######################################################################

1;
