001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.yarn.api.records; 020 021import org.apache.hadoop.classification.InterfaceAudience.Public; 022import org.apache.hadoop.classification.InterfaceStability.Evolving; 023import org.apache.hadoop.classification.InterfaceStability.Unstable; 024import org.apache.hadoop.yarn.util.Records; 025 026/** 027 * {@code LogAggregationContext} represents all of the 028 * information needed by the {@code NodeManager} to handle 029 * the logs for an application. 030 * <p> 031 * It includes details such as: 032 * <ul> 033 * <li> 034 * includePattern. It uses Java Regex to filter the log files 035 * which match the defined include pattern and those log files 036 * will be uploaded when the application finishes. 037 * </li> 038 * <li> 039 * excludePattern. It uses Java Regex to filter the log files 040 * which match the defined exclude pattern and those log files 041 * will not be uploaded when application finishes. If the log file 042 * name matches both the include and the exclude pattern, this file 043 * will be excluded eventually. 044 * </li> 045 * <li> 046 * rolledLogsIncludePattern. It uses Java Regex to filter the log files 047 * which match the defined include pattern and those log files 048 * will be aggregated in a rolling fashion. 049 * </li> 050 * <li> 051 * rolledLogsExcludePattern. It uses Java Regex to filter the log files 052 * which match the defined exclude pattern and those log files 053 * will not be aggregated in a rolling fashion. If the log file 054 * name matches both the include and the exclude pattern, this file 055 * will be excluded eventually. 056 * </li> 057 * <li> 058 * policyClassName. The policy class name that implements 059 * ContainerLogAggregationPolicy. At runtime, nodemanager will the policy 060 * if a given container's log should be aggregated based on the 061 * ContainerType and other runtime state such as exit code by calling 062 * ContainerLogAggregationPolicy#shouldDoLogAggregation. 063 * This is useful when the app only wants to aggregate logs of a subset of 064 * containers. Here are the available policies. Please make sure to specify 065 * the canonical name by prefixing org.apache.hadoop.yarn.server. 066 * nodemanager.containermanager.logaggregation. 067 * to the class simple name below. 068 * NoneContainerLogAggregationPolicy: skip aggregation for all containers. 069 * AllContainerLogAggregationPolicy: aggregate all containers. 070 * AMOrFailedContainerLogAggregationPolicy: aggregate application master 071 * or failed containers. 072 * FailedOrKilledContainerLogAggregationPolicy: aggregate failed or killed 073 * containers 074 * FailedContainerLogAggregationPolicy: aggregate failed containers 075 * AMOnlyLogAggregationPolicy: aggregate application master containers 076 * SampleContainerLogAggregationPolicy: sample logs of successful worker 077 * containers, in addition to application master and failed/killed 078 * containers. 079 * If it isn't specified, it will use the cluster-wide default policy 080 * defined by configuration yarn.nodemanager.log-aggregation.policy.class. 081 * The default value of yarn.nodemanager.log-aggregation.policy.class is 082 * AllContainerLogAggregationPolicy. 083 * </li> 084 * <li> 085 * policyParameters. The parameters passed to the policy class via 086 * ContainerLogAggregationPolicy#parseParameters during the policy object 087 * initialization. This is optional. Some policy class might use parameters 088 * to adjust its settings. It is up to policy class to define the scheme of 089 * parameters. 090 * For example, SampleContainerLogAggregationPolicy supports the format of 091 * "SR:0.5,MIN:50", which means sample rate of 50% beyond the first 50 092 * successful worker containers. 093 * </li> 094 * </ul> 095 * 096 * @see ApplicationSubmissionContext 097 */ 098 099@Evolving 100@Public 101public abstract class LogAggregationContext { 102 103 @Public 104 @Unstable 105 public static LogAggregationContext newInstance(String includePattern, 106 String excludePattern) { 107 LogAggregationContext context = Records.newRecord(LogAggregationContext.class); 108 context.setIncludePattern(includePattern); 109 context.setExcludePattern(excludePattern); 110 return context; 111 } 112 113 @Public 114 @Unstable 115 public static LogAggregationContext newInstance(String includePattern, 116 String excludePattern, String rolledLogsIncludePattern, 117 String rolledLogsExcludePattern) { 118 LogAggregationContext context = 119 Records.newRecord(LogAggregationContext.class); 120 context.setIncludePattern(includePattern); 121 context.setExcludePattern(excludePattern); 122 context.setRolledLogsIncludePattern(rolledLogsIncludePattern); 123 context.setRolledLogsExcludePattern(rolledLogsExcludePattern); 124 return context; 125 } 126 127 @Public 128 @Unstable 129 public static LogAggregationContext newInstance(String includePattern, 130 String excludePattern, String rolledLogsIncludePattern, 131 String rolledLogsExcludePattern, String policyClassName, 132 String policyParameters) { 133 LogAggregationContext context = 134 Records.newRecord(LogAggregationContext.class); 135 context.setIncludePattern(includePattern); 136 context.setExcludePattern(excludePattern); 137 context.setRolledLogsIncludePattern(rolledLogsIncludePattern); 138 context.setRolledLogsExcludePattern(rolledLogsExcludePattern); 139 context.setLogAggregationPolicyClassName(policyClassName); 140 context.setLogAggregationPolicyParameters(policyParameters); 141 return context; 142 } 143 144 /** 145 * Get include pattern. This includePattern only takes affect 146 * on logs that exist at the time of application finish. 147 * 148 * @return include pattern 149 */ 150 @Public 151 @Unstable 152 public abstract String getIncludePattern(); 153 154 /** 155 * Set include pattern. This includePattern only takes affect 156 * on logs that exist at the time of application finish. 157 * 158 * @param includePattern 159 */ 160 @Public 161 @Unstable 162 public abstract void setIncludePattern(String includePattern); 163 164 /** 165 * Get exclude pattern. This excludePattern only takes affect 166 * on logs that exist at the time of application finish. 167 * 168 * @return exclude pattern 169 */ 170 @Public 171 @Unstable 172 public abstract String getExcludePattern(); 173 174 /** 175 * Set exclude pattern. This excludePattern only takes affect 176 * on logs that exist at the time of application finish. 177 * 178 * @param excludePattern 179 */ 180 @Public 181 @Unstable 182 public abstract void setExcludePattern(String excludePattern); 183 184 /** 185 * Get include pattern in a rolling fashion. 186 * 187 * @return include pattern 188 */ 189 @Public 190 @Unstable 191 public abstract String getRolledLogsIncludePattern(); 192 193 /** 194 * Set include pattern in a rolling fashion. 195 * 196 * @param rolledLogsIncludePattern 197 */ 198 @Public 199 @Unstable 200 public abstract void setRolledLogsIncludePattern( 201 String rolledLogsIncludePattern); 202 203 /** 204 * Get exclude pattern for aggregation in a rolling fashion. 205 * 206 * @return exclude pattern 207 */ 208 @Public 209 @Unstable 210 public abstract String getRolledLogsExcludePattern(); 211 212 /** 213 * Set exclude pattern for in a rolling fashion. 214 * 215 * @param rolledLogsExcludePattern 216 */ 217 @Public 218 @Unstable 219 public abstract void setRolledLogsExcludePattern( 220 String rolledLogsExcludePattern); 221 222 /** 223 * Get the log aggregation policy class. 224 * 225 * @return log aggregation policy class 226 */ 227 @Public 228 @Unstable 229 public abstract String getLogAggregationPolicyClassName(); 230 231 /** 232 * Set the log aggregation policy class. 233 * 234 * @param className 235 */ 236 @Public 237 @Unstable 238 public abstract void setLogAggregationPolicyClassName( 239 String className); 240 241 /** 242 * Get the log aggregation policy parameters. 243 * 244 * @return log aggregation policy parameters 245 */ 246 @Public 247 @Unstable 248 public abstract String getLogAggregationPolicyParameters(); 249 250 /** 251 * Set the log aggregation policy parameters. 252 * There is no schema defined for the parameters string. 253 * It is up to the log aggregation policy class to decide how to parse 254 * the parameters string. 255 * 256 * @param parameters 257 */ 258 @Public 259 @Unstable 260 public abstract void setLogAggregationPolicyParameters( 261 String parameters); 262}