给老子爬爬爬！2019国家统计局最新城乡划分代码
作者：互联网
爬一下最新的行政区划
http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/index.html
引入依赖
<!-- https://mvnrepository.com/artifact/com.belerweb/pinyin4j -->
<dependency>
<groupId>com.belerweb</groupId>
<artifactId>pinyin4j</artifactId>
<version>2.5.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.projectlombok/lombok -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.8</version>
<scope>provided</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.13.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp -->
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>4.4.1</version>
</dependency>
代码
package com.demo.tools;
import com.alibaba.fastjson.JSON;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import net.sourceforge.pinyin4j.PinyinHelper;
import net.sourceforge.pinyin4j.format.HanyuPinyinCaseType;
import net.sourceforge.pinyin4j.format.HanyuPinyinOutputFormat;
import net.sourceforge.pinyin4j.format.exception.BadHanyuPinyinOutputFormatCombination;
import okhttp3.ConnectionPool;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
/**
* Created by 小LUA on 2020-03-30 11:39.
*/
@Slf4j
public class GetProvince {
private static final OkHttpClient client = new OkHttpClient.Builder()
.connectTimeout(5, TimeUnit.MINUTES)
.writeTimeout(5, TimeUnit.MINUTES)
.readTimeout(5, TimeUnit.MINUTES)
.connectionPool(new ConnectionPool(0, 30, TimeUnit.MINUTES))
.build();
/**
* 获取首字母
* @param chinese
* @return
*/
private static String getFirstSpell(String chinese) {
try {
HanyuPinyinOutputFormat defaultFormat = new HanyuPinyinOutputFormat();
defaultFormat.setCaseType(HanyuPinyinCaseType.UPPERCASE);
String[] temp = new String[0];
try {
temp = PinyinHelper.toHanyuPinyinStringArray(chinese.charAt(0), defaultFormat);
} catch (BadHanyuPinyinOutputFormatCombination badHanyuPinyinOutputFormatCombination) {
badHanyuPinyinOutputFormatCombination.printStackTrace();
}
return temp[0].charAt(0)+"";
} catch (Exception e){
return "";
}
}
/**
* 读取URL内容
* @param url
* @return
* @throws IOException
*/
public static String readUrl(String url) throws IOException {
System.out.println("读取URL：" + url);
Request request = new Request.Builder()
.url(url)
.build();
Response response = client.newCall(request).execute();
String body = new String(response.body().bytes(), "gb2312");
// System.out.println(body);
return body;
}
public static void main(String[] args) throws Exception {
// 正则
Pattern pattern = Pattern.compile("[0-9]+");
// System.out.println(pattern.matcher("划代码").matches());
// System.out.println(pattern.matcher("110000").matches());
List<Location> all = new ArrayList<>();
String baseUrl = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/";
// 主页
Elements provinceList = null;
do {
String indexContent = readUrl(baseUrl + "index.html");
provinceList = Jsoup.parse(indexContent).getElementsByClass("provincetr");
System.out.println("provinceList是否为空：" + provinceList.isEmpty());
} while (provinceList.isEmpty());
for (Element pElement : provinceList) {
// 获取省名称 + 子地址
Elements a = pElement.select("a");
for (Element e : a) {
String pName = e.text();
String pHref = e.attr("href");
String pCode = pHref.substring(0, pHref.indexOf("."));
String cityUrl = baseUrl + pHref;
System.out.println(pName + "," + pCode + "," + cityUrl);
Long provinceCode = Long.valueOf(pCode + "0000");
// 设置省信息
Location provinceInfo = new Location();
provinceInfo.setCode(provinceCode);
provinceInfo.setName(pName);
provinceInfo.setLevel(1);
provinceInfo.setLetterSort(getFirstSpell(pName));
List<Location> cities = new ArrayList<>();
provinceInfo.setChilds(cities); // 省下面：市
all.add(provinceInfo);
// 读取城市页面
Elements cityList = null;
do {
String cityContent = readUrl(cityUrl);
cityList = Jsoup.parse(cityContent).getElementsByClass("citytr");
System.out.println("cityList是否为空：" + cityList.isEmpty());
} while (cityList.isEmpty());
for (Element cElement : cityList) {
Elements aa = cElement.select("a");
for (Element ee : aa) {
String cName = ee.text();
// 过滤掉比如name为110100000000的数据，只需要取汉字的
if (pattern.matcher(cName).matches()){
continue;
}
String cHref = ee.attr("href");
String cCode = cHref.substring(cHref.indexOf("/")+1, cHref.indexOf("."));
String countyUrl = baseUrl + cHref;
System.out.println(cName + "," + cCode + "," + countyUrl);
Long cityCode = Long.valueOf(cCode + "00");
// 设置城市信息
Location city = new Location();
city.setCode(cityCode);
city.setName(cName);
city.setLevel(2);
city.setLetterSort(getFirstSpell(cName));
List<Location> counties = new ArrayList<>();
city.setChilds(counties); // 市下面：区
city.setParentCode(provinceCode);
cities.add(city); // 添加到城市列表
// 读取区页面
Elements countyList = null;
do{
String countyContent = readUrl(countyUrl);
countyList = Jsoup.parse(countyContent).getElementsByClass("countytr");
if ("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/44/4419.html".equals(countyUrl) ||
"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/44/4420.html".equals(countyUrl) ||
"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/46/4604.html".equals(countyUrl)){
countyList = Jsoup.parse(countyContent).getElementsByClass("towntr");
}
System.out.println("countyList是否为空：" + countyList.isEmpty());
} while (countyList.isEmpty());
for (Element aElement : countyList) {
Elements aaa = aElement.select("a");
for (Element eee : aaa) {
String aName = eee.text();
// 过滤掉比如name为110100000000的数据，只需要取汉字的
if (pattern.matcher(aName).matches()){
continue;
}
String aHref = eee.attr("href");
String aCode = aHref.substring(aHref.indexOf("/")+1, aHref.indexOf("."));
System.out.println(aName + "," + aCode);
// 设置区信息
Location county = new Location();
county.setCode(Long.valueOf(aCode));
county.setName(aName);
county.setLevel(3);
county.setLetterSort(getFirstSpell(aName));
county.setParentCode(cityCode);
counties.add(county); // 添加到区列表
}
}
}
}
}
}
String jsonString = JSON.toJSONString(all);
System.out.println(jsonString);
write(jsonString);
}
public static void write(String str) throws IOException {
FileOutputStream out = new FileOutputStream("2019省市区-大陆.json");
out.write(str.getBytes());
out.flush();
out.close();
}
}
@Getter
@Setter
class Location{
private Long code;
private String name;
private Integer level;
private String letterSort;
private Long parentCode;
private List<Location> childs;
}
爬完数据我只是存在了json文件里了，如果你需要存到数据库，只需要对 all 进行处理即可。或者读文件再处理
private static void read() throws IOException {
FileInputStream in = new FileInputStream("2019省市区-大陆.json");
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
StringBuilder sb = new StringBuilder();
String t;
while ((t = reader.readLine()) != null){
sb.append(t);
}
List<Location> cities = JSONArray.parseArray(sb.toString(), Location.class);
cities.forEach(e -> {
// TODO
});
}
另附：全部数据（很小一部分的名称爬下来就是乱码，导致首字母识别不出来需要手动改正，搜索【"letterSort": ""】）
一共有6个，不算多。
数据文件：https://github.com/Mysakura/DataFiles
标签：String,城乡,System,2019,import,println,new,国家统计局,out 来源： https://www.cnblogs.com/LUA123/p/12603550.html