package com.hfg.util.io;

import java.net.HttpURLConnection;
import java.net.URL;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.hfg.util.collection.CollectionUtil;
import com.hfg.util.StringBuilderPlus;
import com.hfg.util.StringUtil;

//------------------------------------------------------------------------------
/**
 * Deliniates HTTP files matching the specifed path.
 *
 * @author J. Alex Taylor, hairyfatguy.com
 */
//------------------------------------------------------------------------------
// com.hfg XML/HTML Coding Library
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
//
// J. Alex Taylor, President, Founder, CEO, COO, CFO, OOPS hairyfatguy.com
// jataylor@hairyfatguy.com
//------------------------------------------------------------------------------

public class HTTPRemoteFileLister extends AbstractRemoteFileLister<HTTPRemoteFile>
{

   //***************************************************************************
   // PRIVATE FIELDS
   //***************************************************************************

   private List<HTTPRemoteFile>      mRemoteFiles;

   private String mUserAgentString = sDefaultUserAgentString;

   private static String sDefaultUserAgentString;
   private static final String DEFAULT_PROTOCOL = "http";

   //***************************************************************************
   // CONSTRUCTORS
   //***************************************************************************

   //---------------------------------------------------------------------------
   public HTTPRemoteFileLister()
   {
      super();
   }

   //---------------------------------------------------------------------------
   public HTTPRemoteFileLister(URL inFilePath)
   {
      super(inFilePath.toString());
   }

   //---------------------------------------------------------------------------
   public HTTPRemoteFileLister(String inFilePath)
   {
      super(inFilePath);
   }

   //---------------------------------------------------------------------------
   public HTTPRemoteFileLister(String inFilePath, List<RemoteFileFilter> inFilterList)
   {
      super(inFilePath, inFilterList);
   }


   //***************************************************************************
   // PUBLIC METHODS
   //***************************************************************************

   //---------------------------------------------------------------------------
   @Override
   public String getProtocol()
   {
      return (getFilePath() != null ? getFilePath().substring(0, getFilePath().indexOf(":")) : DEFAULT_PROTOCOL);
   }

   //---------------------------------------------------------------------------
   public void clearRemoteFileList()
   {
      mRemoteFiles = null;
   }

   //---------------------------------------------------------------------------
   /**
    Sets the user agent string to use during communication with the server.
    Some sites don't want to talk to Java so some spoofing may be in order.

    @param inValue The user agent string to use.
    */
   public void setUserAgentString(String inValue)
   {
      mUserAgentString = inValue;
   }

   //---------------------------------------------------------------------------
   /**
    Sets the default user agent string to use during communication with the server.
    Some sites don't want to talk to Java so some spoofing may be in order.

    @param inValue The default user agent string to use.
    */
   public static void setDefaultUserAgentString(String inValue)
   {
      sDefaultUserAgentString = inValue;
   }

   //***************************************************************************
   // PROTECTED METHODS
   //***************************************************************************

   //---------------------------------------------------------------------------
   protected List<HTTPRemoteFile> getUnfilteredRemoteFileListImpl()
   {
      if (null == mRemoteFiles)
      {
         mRemoteFiles = new ArrayList<>();

         List<String> urls = expandURL(getFilePath());
         for (String url : urls)
         {
            mRemoteFiles.add(new HTTPRemoteFile(url).setUserAgentString(mUserAgentString));
         }
      }

      return mRemoteFiles;
   }

   //***************************************************************************
   // PRIVATE METHODS
   //***************************************************************************

   //---------------------------------------------------------------------------
   private List<String> expandURL(String inURL)
   {
      List<String> urls = new ArrayList<>();

      int index = inURL.indexOf("*");
      if (index > 0)
      {
         String dirURL = getDirURL(inURL, index);
         List<String> links = getLinksInDir(dirURL);

         String regexp = getDirRegExp(inURL, index);
         String urlEnd = getURLEnd(inURL, index);

         // We don't want to get caught by things like http to https redirection
         Pattern dirPattern = Pattern.compile("(https?://" + dirURL.substring(dirURL.indexOf("://") + 3)+ ")");

         if (CollectionUtil.hasValues(links))
         {
            for (String link : links)
            {
               Matcher m = dirPattern.matcher(link);
               if (m.find())
               {
                  link = link.substring(m.group(1).length());
               }

               if (link.matches(regexp))
               {
                  if (urlEnd.length() > 0)
                  {
                     urls.addAll(expandURL(dirURL + link.substring(0, link.indexOf("/")) + (urlEnd.startsWith("/") ? "" : "/") + urlEnd));
                  }
                  else
                  {
                     urls.add(dirURL + link);
                  }
               }
            }
         }
      }
      else
      {
         urls.add(inURL);
      }

      return urls;
   }

   //---------------------------------------------------------------------------
   private String getDirURL(String inURL, int inIndex)
   {
      String dir = inURL.substring(0, inIndex);
      int slashIndex = dir.lastIndexOf('/');
      if (slashIndex > 0)
      {
         dir = dir.substring(0, slashIndex + 1);
      }

      return dir;
   }

   //---------------------------------------------------------------------------
   private String getDirRegExp(String inURL, int inIndex)
   {
      String dirNameLeftOfWildcard = inURL.substring(0, inIndex);
      int slashIndex = dirNameLeftOfWildcard.lastIndexOf('/');
      if (slashIndex >= 0)
      {
         dirNameLeftOfWildcard = dirNameLeftOfWildcard.substring(slashIndex + 1);
      }

      String dirNameRightOfWildcard = inURL.substring(inIndex);
      slashIndex = dirNameRightOfWildcard.indexOf('/');
      if (slashIndex >= 0)
      {
         dirNameRightOfWildcard = dirNameRightOfWildcard.substring(0, slashIndex);
      }

      String regexp = dirNameLeftOfWildcard + dirNameRightOfWildcard;

      if (slashIndex > 0)
      {
         regexp += "(?:/|" + inURL.substring(inIndex).substring(slashIndex) + "|$)";
      }

      regexp = StringUtil.replaceAll(regexp, ".", "\\.");
      regexp = StringUtil.replaceAll(regexp, "*", ".*");

      return regexp;
   }

   //---------------------------------------------------------------------------
   private String getURLEnd(String inURL, int inIndex)
   {
      String snippet = inURL.substring(inIndex);
      int slashIndex = snippet.indexOf('/');
      if (slashIndex >= 0)
      {
         snippet = snippet.substring(slashIndex);
      }
      else
      {
         snippet = "";
      }

      return snippet;
   }

   //---------------------------------------------------------------------------
   private HttpURLConnection openConnection(String inURL)
      throws IOException
   {
      HttpURLConnection conn = (HttpURLConnection) new URL(inURL).openConnection();

      // Pretend to be a browser. Some sites don't want to talk to Java
      if (StringUtil.isSet(mUserAgentString))
      {
         conn.setRequestProperty("User-Agent", mUserAgentString);
      }

      if (getCredentials() != null)
      {
         String base64Auth = Base64.getEncoder().encodeToString((getCredentials().getUser() + ":" + new String(getCredentials().getPassword())).getBytes());
         conn.setRequestProperty("Authorization", "Basic "+ base64Auth);
      }

      conn.connect();

      return conn;
   }

   //---------------------------------------------------------------------------
   private List<String> getLinksInDir(String inDirURL)
   {
      List<String> links = new ArrayList<>();

      try
      {
         String url = inDirURL;

         HttpURLConnection conn = null;
         int numAttempts = 0;
         while (numAttempts < 3)
         {
            conn = openConnection(url);
            if (conn.getResponseCode() == HttpURLConnection.HTTP_MOVED_PERM
                || conn.getResponseCode() == HttpURLConnection.HTTP_MOVED_PERM)
            {
               url = conn.getHeaderField("Location");
               numAttempts++;
            }
            else
            {
               break;
            }
         }


//         System.out.println("RESPONSE CODE: " + conn.getResponseCode());
//         System.out.println("RESPONSE MSG: " + conn.getResponseMessage());
//         System.out.println("CONTENT LENGTH: " + conn.getContentLength());
//         System.out.println("CONTENT TYPE: " + conn.getContentType());

         if (conn.getResponseCode() == HttpURLConnection.HTTP_OK)
         {
            BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
            String line;
            StringBuilderPlus dirHTML = new StringBuilderPlus();
            while ((line = reader.readLine()) != null)
            {
               dirHTML.appendln(line);
            }
            reader.close();

            // Painful but we need to scrape the page looking for the directory's contents.
            // (Assuming it IS a directory.)
            Pattern p = Pattern.compile("<a.+?href=[\\\"\\\'](.+?)[\\\"\\\']",
                                        Pattern.CASE_INSENSITIVE);

            Matcher m = p.matcher(dirHTML);
            while (m.find())
            {
               links.add(m.group(1));
            }
         }
      }
      catch (IOException e)
      {
         throw new RuntimeException(e.toString());
      }

      return links;
   }
}