驗證資料庫中URL的有效性
曲庫中一些歌曲的URL雖然存在,但是根據URL已經下載不到音樂了.
Nginx顯示404錯誤.
驗證資料庫中歌曲的URL是否能夠下載
首先,先把資料庫中的歌曲URL匯出到檔案.如下格式(歌曲ID,音樂地址型別,路徑)
1000;AccompanimentURL ;/00/00/00001000_accompaniment.m4a
然後使用程式掃描URL
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.7</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.7</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.1.0.Final</version>
</dependency>
程式比較渣..以後得學習一下物件封裝
Nginx顯示404錯誤.
驗證資料庫中歌曲的URL是否能夠下載
首先,先把資料庫中的歌曲URL匯出到檔案.如下格式(歌曲ID,音樂地址型別,路徑)
1000;AccompanimentURL ;/00/00/00001000_accompaniment.m4a
然後使用程式掃描URL
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.7</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.7</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.1.0.Final</version>
</dependency>
程式比較渣..以後得學習一下物件封裝
- import java.io.BufferedReader;
- import java.io.FileInputStream;
- import java.io.FileWriter;
- import java.io.IOException;
- import java.io.InputStreamReader;
- import java.util.HashMap;
- import java.util.HashSet;
- import java.util.Map;
- import java.util.Set;
- import java.util.concurrent.BlockingQueue;
- import java.util.concurrent.LinkedBlockingQueue;
- import java.util.concurrent.Semaphore;
- import java.util.concurrent.atomic.AtomicInteger;
- import io.netty.bootstrap.Bootstrap;
- import io.netty.buffer.ByteBuf;
- import io.netty.buffer.Unpooled;
- import io.netty.channel.Channel;
- import io.netty.channel.ChannelHandlerContext;
- import io.netty.channel.ChannelInboundHandlerAdapter;
- import io.netty.channel.ChannelInitializer;
- import io.netty.channel.EventLoopGroup;
- import io.netty.channel.nio.NioEventLoopGroup;
- import io.netty.channel.socket.nio.NioSocketChannel;
- import io.netty.handler.codec.LineBasedFrameDecoder;
- import io.netty.handler.codec.string.StringDecoder;
- public class URLChecker {
- public static void main(String[] args) {
- String sourceFile="F:\\normal.txt";
- String resultFile="F:\\result.csv";
- new URLInput(sourceFile, resultFile);
- }
- }
- class URLConnection extends Thread{
- @Override
- public void run() {
- while(true){
- int start=count.get();
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- int end=count.get();
- System.out.println("每秒檢查:"+(end-start));
- }
- }
- Semaphore sem = new Semaphore(20);
- String[] hosts = new String[5];
- AtomicInteger count = new AtomicInteger();
- EventLoopGroup group = new NioEventLoopGroup(3);
- URLInput input;
- URLOutput output;
- URLConnection(URLInput input,URLOutput writer) {
- hosts[0] = "172.16.1.151";
- hosts[1] = "172.16.1.152";
- hosts[2] = "172.16.1.153";
- hosts[3] = "172.16.1.154";
- hosts[4] = "172.16.1.155";
- this.output=writer;
- this.input=input;
- }
- public void connection(final Map<String, String> map)
- throws InterruptedException {
- sem.acquire();
- int index = count.getAndIncrement();
- Bootstrap boot = new Bootstrap();
- boot.group(group).channel(NioSocketChannel.class).handler(new ChannelInitializer() {
- @Override
- protected void initChannel(Channel ch) throws Exception {
- ch.pipeline().addLast(new LineBasedFrameDecoder(409600));
- ch.pipeline().addLast(new StringDecoder());
- ch.pipeline().addLast(new HttpClientHandler(map, sem,input,output));
- }
- });
- boot.connect(hosts[index % hosts.length], 80);
- }
- }
- class HttpClientHandler extends ChannelInboundHandlerAdapter {
- StringBuffer sb = new StringBuffer(512);
- Map<String, String> map = new HashMap<String, String>();
- Semaphore sem;
- URLInput input;
- URLOutput writer;
- public HttpClientHandler(Map<String, String> map, Semaphore sem, URLInput input, URLOutput writer) {
- this.map = map;
- this.sem = sem;
- this.writer=writer;
- this.input=input;
- }
- @Override
- public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
- sem.release();
- ctx.close();
- writer.addSuccURL(map);
- }
- @Override
- public void channelActive(ChannelHandlerContext ctx) throws Exception {
- StringBuilder sb = new StringBuilder();
- sb.append("HEAD " + map.get("url") + " HTTP/1.0\r\n");
- sb.append("HOST:" + 80 + "\r\n");
- sb.append("Accept:*/*\r\n");
- sb.append("\r\n");
- ByteBuf bb = Unpooled.copiedBuffer(sb.toString().getBytes("utf8"));
- ctx.writeAndFlush(bb);
- }
- @Override
- public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
- String content = (String) msg;
- if (content.contains(":")) {
- String[] s = content.split(":");
- map.put(s[0].trim(), s[1].trim());
- } else if (content.startsWith("HTTP/1.1")) {
- map.put("httpcode", content.replaceAll("HTTP/1.1 ", ""));
- }
- }
- @Override
- public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
- cause.printStackTrace();
- ctx.close();
- input.addFailUrl(map);
- sem.release();
- }
- }
- class URLInput {
- URLConnection urlcon;
- URLInput(String sourceFile, String resultFile) {
- URLOutput output=new URLOutput(resultFile);
- urlcon=new URLConnection(this, output);
- output.start();
- urlcon.start();
- try {
- init(resultFile);
- read(sourceFile);
- } catch (IOException e) {
- e.printStackTrace();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- /**
- * 初始化已經處理的檔案,用於中斷處理後的恢復執行
- *
- * @param resultFile
- * @throws IOException
- */
- private void init(String resultFile) throws IOException {
- BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(resultFile)));
- String row = null;
- while ((row = br.readLine()) != null) {
- String[] data = row.split(",");
- set.add(data[0]);
- }
- br.close();
- }
- public void addFailUrl(Map<String, String> map) {
- failq.add(map);
- }
- final BlockingQueue<Map<String, String>> failq = new LinkedBlockingQueue<Map<String, String>>();
- Set<String> set = new HashSet<String>();
- private void read(String sourceFile) throws IOException, InterruptedException {
- BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(sourceFile)));
- String row = null;
- while ((row = br.readLine()) != null) {
- while (failq.size() != 0) {
- final Map<String, String> m = failq.take();
- urlcon.connection(m);
- }
- String[] data = row.split(";");
- final Map<String, String> map = new HashMap<String, String>();
- map.put("songid", data[0]);
- map.put("type", data[1]);
- map.put("url", data[2]);
- if (!set.contains(data[0])) {
- urlcon.connection(map);
- }
- }
- br.close();
- System.out.println("Finish!!");
- }
- }
- class URLOutput extends Thread {
- BlockingQueue<Map<String, String>> succq = new LinkedBlockingQueue<Map<String, String>>();
- String resultFile;
- public void addSuccURL(Map<String, String> map) {
- succq.add(map);
- }
- public URLOutput(String resultFile) {
- this.resultFile = resultFile;
- }
- @Override
- public void run() {
- Map<String, String> map = null;
- FileWriter fw = null;
- try {
- fw = new FileWriter(resultFile, true);
- while ((map = succq.take()) != null) {
- fw.write(map.get("songid") + "," + map.get("type") + "," + map.get("url") + "," + map.get("httpcode")
- + "," + map.get("Content-Length") + "\n");
- }
- } catch (InterruptedException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- try {
- fw.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
-
}
來自 “ ITPUB部落格 ” ,連結:http://blog.itpub.net/29254281/viewspace-2142047/,如需轉載,請註明出處,否則將追究法律責任。
相關文章
- 驗證oracle 11g資料庫的RMAN備份的有效性Oracle資料庫
- RMAN 驗證 資料檔案 和 備份 的有效性
- 資料庫的身份驗證方式資料庫
- SQL資料庫開發中的SSIS 延遲驗證方法SQL資料庫
- winform中輸入資料的驗證ORM
- JavaScript 如何驗證 URLJavaScript
- 通過shell指令碼批量驗證dataguard的有效性指令碼
- 騰訊資料庫tdsql部署與驗證資料庫SQL
- JS驗證URL正則JS
- 如何驗證Oracle資料庫中表的型別Oracle資料庫型別
- Pydantic:強大的Python 資料驗證庫Python
- 驗證使用者輸入的口令是否與資料庫中的匹配資料庫
- oracle11g rman驗證備份有效性Oracle
- 如何提高少樣本學習中的引數有效性以及資料有效性
- angular中的表單資料自定義驗證Angular
- 關於資料庫管理員的驗證小結資料庫
- Oracle23ai 資料庫的簡單驗證OracleAI資料庫
- 在資料庫中如何驗證 In-Memory 選項的使用 (文件 ID 2178917.1)資料庫
- Oracle資料庫密碼延遲驗證Oracle資料庫密碼
- 從資料庫中修改zabbix的驗證方式 0 內建 1 LDAP 2 HTTP資料庫LDAHTTP
- jsonlint:python的json資料驗證庫JSONPython
- 基於外部OS驗證的資料庫使用者資料庫
- 常用資料庫 JDBC URL 格式資料庫JDBC
- 1.6.3. 資料庫管理員身份驗證方法資料庫
- javascript 資料型態/結構驗證庫 : SkeletonsJavaScript
- 11g rman validate驗證資料庫資料庫
- 【詳細、開箱即用】.NET企業微信回撥配置(資料回撥URL和指令回撥URL驗證)
- 不存資料庫生成驗證碼(totp演算法)資料庫演算法
- 驗證資料是否存在
- 表單資料驗證
- 【OCR】OCR備份策略及備份介質有效性驗證
- 2.4.4 Step 3: 選擇資料庫管理員的身份驗證方法資料庫
- 1.6.3.1. 關於資料庫管理員身份驗證方法資料庫
- data型別的Url格式:把小資料直接嵌入到Url中型別
- Excel 資料有效性如何限定輸入資料?Excel
- 有關資料驗證的原則
- Excel2007中更改資料有效性設定及“資料有效性”命令不可用問題分析Excel
- 使用jdonframework+jboss配置基於jaas的url驗證Framework