import java.awt.Graphics2D;
import java.awt.image.BufferedImage;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Iterator;
import javax.imageio.ImageIO;
public class ProductImporter {
/**
* @param args
*/
public static void main(String[] args) {
boolean jump = true;
try {
BufferedWriter writer = new BufferedWriter(new FileWriter(
"d:\\products.txt"));
ArrayList set = getFatherCat();
Iterator fatherCat = set.iterator();
while (fatherCat.hasNext()) {
System.out.println(fatherCat.next());
String strFatherCat = (String) fatherCat.next();
if ((jump == true)
&& strFatherCat.indexOf("Timepieces and Electronics") > 0)
jump = false;
if (jump == true)
continue;
String mainCatName = strFatherCat
.substring("product_enter.asp?id=".length());
strFatherCat = strFatherCat.replace(" ", "%20");
ArrayList subSet = getSubCat(strFatherCat);
Iterator it = subSet.iterator();
while (it.hasNext()) {
String subCatStr = (String) it.next();
int endPos = subCatStr.indexOf("&bigid=");
String subCatName = subCatStr.substring(
"productdetail.asp?smallid=".length(), endPos);
System.out.println(subCatStr);
subCatStr = subCatStr.replace(" ", "%20");
ArrayList productList = GetProductsList(subCatStr);
for (int i = 0; i < productList.size(); i++) {
GenerateProductSQL(productList, i, mainCatName,
subCatName, writer);
}
}
}
writer.close();
} catch (Exception e) {
e.printStackTrace();
// TODO: handle exception
}
}
/*
*
*/
private static void GenerateProductSQL(ArrayList productList, int i,
String mainCatName, String subCatName, BufferedWriter writer)
throws MalformedURLException, IOException, InterruptedException {
String productStr = (String) productList.get(i);
productStr = productStr.replace(" ", "%20");
System.out.println(productStr);
String productsDetailtUrl = "" + productStr;
String product_name = "";
String product_number = "";
String descripton = "";
String product_image = "";
String quantity_per_caton = "";
String volumn_per_caton = "";
URL url = new URL(productsDetailtUrl);
// Read all the text returned by the server
BufferedReader in = new BufferedReader(new InputStreamReader(url
.openStream()));
String str;
while ((str = in.readLine()) != null) {
// Get Item id
//
if (str.indexOf("color=#666666>Item") >= 0) {
// jump over 2 rows
str = in.readLine();
str = in.readLine();
if (str.indexOf("class=font10>") >= 0) {
int startPos = str.indexOf("class=font10>")
+ "class=font10>".length();
int endPos = str.indexOf("");
product_number = str.substring(startPos, endPos);
}
}
// Get Product name
//
if (str.indexOf("#666666>Prouduct name :") >= 0) {
// jump over 2 rows
str = in.readLine();
if (str.indexOf("class=font10>") >= 0) {
int startPos = str.indexOf("class=font10>")
+ "class=font10>".length();
int endPos = str.indexOf("");
product_name = str.substring(startPos, endPos);
}
}
// Get Description
//
if (str.indexOf("666666>Description") >= 0) {
// jump over 1 rows
str = in.readLine();
str = in.readLine();
descripton = "";
if (str.indexOf("class=font10>") >= 0) {
int startPos = str.indexOf("class=font10>")
+ "class=font10>".length();
int endPos = str.indexOf("");
if (endPos < 0) {
descripton = str.substring(startPos);
str = in.readLine();
while (str.indexOf("") < 0) {
if (str.indexOf("") < 0)
descripton += str;
str = in.readLine();
}
endPos = str.indexOf("");
descripton += str.substring(0, endPos);
} else {
descripton = str.substring(startPos, endPos);
}
}
}
// Quantity/Carton
//
if (str.indexOf("66666>Quantity") >= 0) {
// jump over 2 rows
str = in.readLine();
str = in.readLine();
if (str.indexOf("class=font10>") >= 0) {
int startPos = str.indexOf("class=font10>")
+ "class=font10>".length();
int endPos = str.indexOf("");
quantity_per_caton = str.substring(startPos, endPos);
}
}
// Volume/Carton
//
if (str.indexOf("Volume/Carton (m3) := 0) {
// jump over 2 rows
str = in.readLine();
str = in.readLine();
if (str.indexOf("class=font10>") >= 0) {
int startPos = str.indexOf("class=font10>")
+ "class=font10>".length();
int endPos = str.indexOf("");
volumn_per_caton = str.substring(startPos, endPos);
}
}
if (str.indexOf("UploadFiles/") >= 0) {
int startPos = str.indexOf("UploadFiles/");
int endPos = str.indexOf("\">", (startPos + 1));
product_image = str.substring(startPos
+ "UploadFiles/".length(), endPos);
URL imageUrl = new URL(""
+ str.substring(startPos, endPos));
java.awt.Image image = java.awt.Toolkit.getDefaultToolkit()
.createImage(imageUrl);
while (image.getWidth(null) == -1)
;
// Save picture
//
int w = image.getWidth(null);
int h = image.getHeight(null);
Thread.sleep(4000);
BufferedImage bi = new BufferedImage(w, h,
BufferedImage.TYPE_INT_RGB);
Graphics2D g2 = bi.createGraphics();
g2.drawImage(image, 0, 0, null);
g2.dispose();
Thread.sleep(300);
ImageIO
.write(bi, "jpg",
new File("d:\\pics\\" + product_image));
}
}
/*
* System.out.println( );
*/
writer.write("/#" + mainCatName + "/#" + subCatName + "/#"
+ product_number + "/#" + product_name + "/#" + product_image
+ "/#" + descripton + "/#" + quantity_per_caton + "/#"
+ volumn_per_caton + "/#\n");
writer.flush();
}
private static ArrayList GetProductsList(String subCatStr)
throws MalformedURLException, IOException {
ArrayList productList = new ArrayList();
String productsListUrl = "" + subCatStr;
URL url = new URL(productsListUrl);
// Read all the text returned by the server
BufferedReader in = new BufferedReader(new InputStreamReader(url
.openStream()));
String str;
while ((str = in.readLine()) != null) {
if (str.indexOf("href=product_detail.asp") >= 0) {
int startPos = str.indexOf("product_detail.asp");
int endPos = str.indexOf(" target=_blank", (startPos + 1));
if (endPos < 0)
endPos = str.indexOf("\" >", (startPos + 1));
productList.add(str.substring(startPos, endPos));
}
if (str.indexOf("page=") > 0) {
int startPos = str.indexOf("productdetail.asp?smallid");
int endPos = str.indexOf("\" >[");
String productPageUrl = ""
+ str.substring(startPos, endPos);
productPageUrl = productPageUrl.replace(" ", "%20");
URL pageURL = new URL(productPageUrl);
// Read all the text returned by the server
BufferedReader pageIn = new BufferedReader(
new InputStreamReader(pageURL.openStream()));
String pageStr;
while ((pageStr = pageIn.readLine()) != null) {
if (pageStr.indexOf("href=product_detail.asp") >= 0) {
startPos = pageStr.indexOf("product_detail.asp");
endPos = pageStr.indexOf(" target=_blank",
(startPos + 1));
if (endPos < 0)
endPos = pageStr.indexOf("\" >", (startPos + 1));
productList.add(pageStr.substring(startPos, endPos));
}
}
}
}
return productList;
}
private static ArrayList getSubCat(String fatherCat) {
ArrayList subSet = new ArrayList();
try {
String subCat = "" + fatherCat;// URLEncoder.encode(fatherCat.next(),"iso-8859-1");
URL url = new URL(subCat);
// Read all the text returned by the server
BufferedReader in = new BufferedReader(new InputStreamReader(url
.openStream()));
String str;
while ((str = in.readLine()) != null) {
if (str.indexOf("productdetail.asp") >= 0) {
int startPos = str.indexOf("productdetail.asp");
int endPos = str.indexOf("\">", (startPos + 1));
// System.out.println(str.substring(startPos, endPos));
subSet.add(str.substring(startPos, endPos));
}
}
} catch (Exception e) {
e.printStackTrace();
// TODO: handle exception
}
return subSet;
}
private static ArrayList getFatherCat() {
ArrayList set = new ArrayList();
try {
// TODO Auto-generated method stub
// Send data
URL url = new URL("");
// Read all the text returned by the server
BufferedReader in = new BufferedReader(new InputStreamReader(url
.openStream()));
String str;
while ((str = in.readLine()) != null) {
if (str.indexOf("product_enter.asp") >= 0) {
int startPos = str.indexOf("product_enter.asp");
int endPos = str.indexOf("\">", (startPos + 1));
set.add(str.substring(startPos, endPos));
}
}
} catch (Exception e) {
// TODO: handle exception
}
return set;
}
}
