【本文仅供参考非唯一答案】其中需求实现的方式有多种提供的代码并非唯一写法选择适合的方式即可。

手机日志分析需求数据生成工具类模拟数据生成类MapReduce程序需求编写

手机日志分析需求

编写数据生成器生成1G~10G大小的数据字段必须包括id,日期,手机号码、型号、操作系统字段。

需要将手机号码4~9为掩码处理。

分析2021年、2022年操作系统市场占比、手机型号市场占比情况分析2022年手机运营商市场占比情况分析数据存储到HDFS集群/ana/phone节点下面将分析结果存储到Mysql,并进行数据可视化

数据生成工具类

手机号码随机生成可以采用随机数生成的方式结合三大运营商的号码前三位数为规则进行生成

代码如下

134、135、136、137、138、139、147、150、151、152、157、158、159、172、178、182、183、184、187、188、198、1703、1705、1706*

130、131、132、145、155、156、166、171、175、176、185、186、1704、1707、1708、1709*

中国电信手机号段*

133、153、173、177、180、181、189、191、193、199、1700、1701、1702*

腾讯云API

https://market.cloud.tencent.com/products/31101*/

public

generatePhoneNumbers(1);System.out.println(phoneNumbers);}public

static

CHINA_MOBILE_PREFIX[random.nextInt(CHINA_MOBILE_PREFIX.length)];break;case

1:prefix

CHINA_UNICOM_PREFIX[random.nextInt(CHINA_UNICOM_PREFIX.length)];break;default:prefix

CHINA_TELECOM_PREFIX[random.nextInt(CHINA_TELECOM_PREFIX.length)];}phoneNumber

prefix

replaceCharacters(phoneNumber,3,8);}private

static

{sb.append(random.nextInt(10));}return

sb.toString();}

【采用接口分析】这里可以使用鹅厂或者其他厂商开发的接口进行运营商识别

例子如下

SecretKeySpec(secretKey.getBytes(UTF-8),

mac.getAlgorithm());mac.init(sKey);byte[]

hash

mac.doFinal(signStr.getBytes(UTF-8));String

sig

BASE64Encoder().encode(hash);String

auth

{sb.append();}sb.append(String.format(%s%s,URLEncoder.encode(entry.getKey().toString(),

UTF-8),URLEncoder.encode(entry.getValue().toString(),

UTF-8)));}return

Calendar.getInstance();SimpleDateFormat

sdf

Locale.US);sdf.setTimeZone(TimeZone.getTimeZone(GMT));String

datetime

datetime);headers.put(Authorization,

auth);//

String();queryParams.put(mobile,XXX);//

body参数MapString,

https://service-8c43o60c-1253285064.gz.apigw.tencentcs.com/release/sms;if

{url

urlencode(queryParams);}BufferedReader

null;try

realUrl.openConnection();conn.setConnectTimeout(5000);conn.setReadTimeout(5000);conn.setRequestMethod(method);//

request

{conn.setRequestProperty(entry.getKey(),

request

{conn.setRequestProperty(Content-Type,

application/x-www-form-urlencoded);conn.setDoOutput(true);DataOutputStream

out

DataOutputStream(conn.getOutputStream());out.writeBytes(urlencode(bodyParams));out.flush();out.close();}//

new

InputStreamReader(conn.getInputStream()));String

line;String

line;}System.out.println(result);}

catch

{System.out.println(e);e.printStackTrace();}

finally

1.分析2021年、2022年操作系统市场占比、手机型号市场占比情况*

public

FileWriter(data/phone.log));for

(int

UUID.randomUUID().toString();String

date

PhoneNumberGenerator.generatePhoneNumbers(1);String

model

getRandomOperatingSystem();String

line

operatingSystem;writer.write(line);writer.newLine();}writer.close();}

catch

{phoneNumber.append(random.nextInt(10));}return

phoneNumber.toString();}private

static

Random().nextInt(models.length)];}private

static

Random().nextInt(operatingSystems.length)];}

}结果如下

分析2021年、2022年操作系统市场占比、手机型号市场占比情况

/***

java.io.InputStreamReader;import

org.apache.hadoop.conf.Configuration;

import

org.apache.hadoop.fs.FSDataInputStream;

import

org.apache.hadoop.fs.FileSystem;

import

org.apache.hadoop.io.DoubleWritable;

import

org.apache.hadoop.mapreduce.Job;

import

org.apache.hadoop.mapreduce.Mapper;

import

org.apache.hadoop.mapreduce.Partitioner;

import

org.apache.hadoop.mapreduce.Reducer;

import

org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import

org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public

操作系统市场占比word.set(fields[1].substring(0,

-OS-

手机型号市场占比word.set(fields[1].substring(0,

-Model-

yearTotalCount;result.set(percentage);context.write(key,

result);}}public

Path(data/phone.log);FSDataInputStream

inputStream

InputStreamReader(fs.open(inputPath))))

{String

analysis);job.setJarByClass(PhoneOSAnalysis.class);job.setMapperClass(TokenizerMapper.class);//

设置自定义分区器job.setPartitionerClass(CustomPartitioner.class);job.setNumReduceTasks(2);job.setReducerClass(MarketShareReducer.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(DoubleWritable.class);Path

outputPath

true);}FileInputFormat.addInputPath(job,

new

Path(data/phone.log));FileOutputFormat.setOutputPath(job,

new

Path(String.valueOf(outputPath)));//

TextInputFormat.addInputPath(job,

new

Path(hdfs://192.168.192.100:8020/));

TextInputFormat.outInputPath(job,

new

Path(hdfs://192.168.192.100:8020/));System.exit(job.waitForCompletion(true)

1);}public

(key.toString().contains(2021))

{return

org.apache.hadoop.conf.Configuration;

import

org.apache.hadoop.fs.FileSystem;

import

org.apache.hadoop.io.DoubleWritable;

import

org.apache.hadoop.mapreduce.Job;

import

org.apache.hadoop.mapreduce.Mapper;

import

org.apache.hadoop.mapreduce.Reducer;

import

org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import

org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public

class

手机运营商市场占比word.set(fields[1].substring(0,

-Operator-

getCarrier(fields[2]));context.write(word,

one);}}private

lineCount2022;result.set(percentage);context.write(key,

result);}}public

InputStreamReader(fs.open(inputPath))))

{String

PhoneOperator);job.setJarByClass(OperatorMR.class);job.setMapperClass(TokenizerMapper.class);job.setReducerClass(MarketShareReducer.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(DoubleWritable.class);Path

outputPath

TextInputFormat.addInputPath(job,

new

Path(hdfs://192.168.192.100:8020/));

TextInputFormat.outInputPath(job,

new

Path(hdfs://192.168.192.100:8020/));FileInputFormat.addInputPath(job,

new

Path(data/phone.log));FileOutputFormat.setOutputPath(job,

outputPath);System.exit(job.waitForCompletion(true)

1);}

java.sql.PreparedStatement;import

org.apache.hadoop.conf.Configuration;

import

org.apache.hadoop.fs.FileSystem;

import

org.apache.hadoop.io.DoubleWritable;

import

org.apache.hadoop.io.NullWritable;

import

org.apache.hadoop.mapreduce.Job;

import

org.apache.hadoop.mapreduce.Mapper;

import

org.apache.hadoop.mapreduce.Reducer;

import

org.apache.hadoop.mapreduce.lib.db.DBConfiguration;

import

org.apache.hadoop.mapreduce.lib.db.DBOutputFormat;

import

org.apache.hadoop.mapreduce.lib.input.FileInputFormat;public

class

手机运营商市场占比word.set(fields[1].substring(0,

-Operator-

getCarrier(fields[2]));context.write(word,

one);}}private

lineCount2022;result.set(percentage);context.write(new

DBOutputWritable(key.toString(),

result.get()),

jdbc:mysql://localhost:3306/blog;String

dbUsername

Admin2022!;DBConfiguration.configureDB(conf,

dbUrl,

DriverManager.getConnection(dbUrl,

dbUsername,

connection.prepareStatement(createTableSql);preparedStatement.executeUpdate();}FileSystem

inputPath

InputStreamReader(fs.open(inputPath))))

{String

PhoneOperator);job.setJarByClass(OPMysqlMR.class);job.setMapperClass(TokenizerMapper.class);job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(DoubleWritable.class);job.setReducerClass(MarketShareReducer.class);job.setOutputKeyClass(DBOutputWritable.class);job.setOutputValueClass(NullWritable.class);//

设置数据库输出DBOutputFormat.setOutput(job,

operator,

market_share);FileInputFormat.addInputPath(job,

new

Path(data/phone.log));System.exit(job.waitForCompletion(true)

1);}

org.apache.hadoop.mapred.lib.db.DBWritable;import

java.io.DataInput;

in.readDouble();}Overridepublic

void

{out.writeUTF(operator);out.writeDouble(market_share);}Overridepublic

void

不需要实现此方法因为我们只会写入数据到数据库}Overridepublic

void

{preparedStatement.setString(1,

operator);preparedStatement.setDouble(2,

market_share);}

org.jfree.data.general.DefaultPieDataset;public

class

jdbc:mysql://localhost:3306/blog;String

dbUsername

DriverManager.getConnection(dbUrl,

dbUsername,

connection.createStatement();ResultSet

resultSet

resultSet.getString(operator);double

marketShare

resultSet.getDouble(market_share);dataset.setValue(operator,

marketShare);}}

ChartFactory.createPieChart(运营商市场占比,

图表标题dataset,

12);pieChart.getTitle().setFont(font);pieChart.getLegend().setItemFont(font);PiePlot

plot

pieChart.getPlot();plot.setLabelFont(font);//

添加饼图到面板并显示ChartPanel

ChartPanel(pieChart);setContentPane(chartPanel);}public

static

PieChartExample();pieChartExample.setSize(600,

600);pieChartExample.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);pieChartExample.setVisible(true);}

}结果如下